VirtualBox creates serial ports as unix domain sockets, which cannot be used with terminal emulators. Use socat to bind them to ptys, and Gnu Screen can deal with them just fine.

VBoxManage modifyvm <vmname> --uartmode1 server ~/.VirtualBox/serial/<vmname>-S0 # or 1 for uart 2
VBoxManage startvm <vmname>
socat ~/.VirtualBox/serial/<vmname>-S0 pty,link=~/.VirtualBox/serial/<vmname>-S0.pty

In screen, attach to serial port using screen command entry mode

screen ~/.VirtualBox/serial/<vmname>-S0.pty

Here's a script that will watch a directory and attach any serial ports it finds to a screen instance: Requires sysutils/fileschanged

#!/bin/sh

# Screen instance identifier
screen="serialer"

# Socket directory
sock_dir=/home/cyberleo/.VirtualBox/serial-sock

# PTY directory
pty_dir=/home/cyberleo/.VirtualBox/serial

# Serial logs directory
log_dir=/home/cyberleo/.VirtualBox/serial-logs

# Make sure they exist
mkdir -p "${sock_dir}" "${pty_dir}" "${log_dir}"

# Clear out dead screens
screen -wipe

# If not inside screen, try to attach; if attach fails, try to launch
if [ -z "${STY}" ]
then
  if ! screen -S "${screen}" -x
  then
    screen -S "${screen}" -d -m -t "Serial" "${0}" "${@}"
    sleep 0.25
    screen -s "${screen}" -x
  fi
  exit $?
fi

# Check if this socket already has a socat running for it
is_serial_bound() {
  name="${1}"
  ps opid,command -U "${USER}" | grep '[s]ocat' | grep -q " ${sock_dir}/${name} "
  return $?
}

# Bind a socket to a pty using socat, and attach a screen to it
bind_serial() {
  name="${1}"
  sock="${2:-${sock_dir}/${name}}"
  pty="${3:-${pty_dir}/${name}.pty}"

  set -x
  # Spawn socat to attach socket to pty
  socat "${sock}" pty,link="${pty}" &
  sleep 0.25

  # Start new screen
  screen -S "${screen}" -X screen -t "${name}" "${pty}"
  # Set up logging
  screen -S "${screen}" -X logfile "${log_dir}/${name}.log"
  screen -S "${screen}" -X log on
  screen -S "${screen}" -X logfile flush 1
  screen -S "${screen}" -X logtstamp on
  set +x
}

# Main loop
( ls -1 "${sock_dir}/"* | sort ; fileschanged -s created -f "${sock_dir}" ) | while read sock
do
  printf "New: %s " "${sock}"

  # Ignore anything that is not a socket
  if [ ! -S "${sock}" ]
  then
    echo "Not a socket."
    continue
  fi

  # Ignore already bound serials
  if is_serial_bound
  then
    echo "Already bound."
    continue
  fi
  echo "Binding..."

  # Bind
  bind_serial "${sock##*/}"

  # Rapidfiring commands to screen doesn't work so well
  sleep 1
done