Work In Progress!

Create a multi-homed router, based off FreeBSD 6.x

  • Support load-balancing NAT outgoing connections across all four routers
  • Support proper routing of incoming connections
  • No ISP support available (No bundling/multilink, no BGP, etc..)
  • PacketFilter preferred

Problems:

  • PacketFilter can NAT outgoing connections properly, but incoming connections always use default gateway.
  • Statefulness means egress packets don't touch the gateway selection rules in PF
  • PF is 'closer' to the device than IPFW, so IPFW only sees untranslated egress packets
  • Need some way to reinject translated packets, so that they can hit the gateway selection rules.

    • All of the above fixed with dummygw (see below)
  • FTP definitely refuses to work, except for passive FTP on a single-IP binatted private machine.

    • Must be bound to a single IP
  • Occasional, hard to trace hangs in certain flows (for instance, microsoft.com's homepage)
  • Totally doesn't handle single-link failure. Failover script must be crafted.
  • DNS works to everywhere except ISP's DNS servers. Why?
  • Too much traffic causes, almost immediately, 'No buffer space available'

    • Restarting uplink subsystem (mpd and dummygw) fixes this for a while.

MPD - Multilink PPP Daemon

  • Easy to set up under FreeBSD
  • Supports multilink in case ISP gets heads out of asses
  • Uses netgraph nodes. All encapsulation done in kernelspace, so it's faster (no context switches)
  • Must use custom up-script to ensure:

    • All six static IPs are added to the node
    • None of the added IPs peers are the same (routing error)
    • We can specify any arbitrary peer gateway, as it is merely an identifier to pick the appropriate link and doesn't affect the traffic routing.

mpd.conf:

default:
        set login admin
        load sbcdsl_common
        load sbcdsl1
        load sbcdsl2
sbcdsl_common:
        set iface disable on-demand
        set iface idle 0
        set bundle enable multilink
        set link no acfcomp protocomp
        set link disable pap chap
        set link accept chap
        set link mtu 1492
        set link keep-alive 10 60
        set link ipcp yes vjcomp
        set ipcp ranges 0.0.0.0/0 0.0.0.0/0
        set iface up-script /usr/local/etc/mpd/mpd.linkup
sbcdsl1:
        new -i ng0 sbcdsl1 sbc1
        set iface addrs 10.0.0.1 10.0.0.2
        set bundle authname "blah1@sbcglobal.net"
        set iface up-script "/usr/local/etc/mpd/mpd.linkup"
        open iface
sbcdsl2:
        new -i ng1 sbcdsl2 sbc2
        set iface addrs 10.0.0.3 10.0.0.4
        set bundle authname "blah2@sbcglobal.net"
        set iface up-script "/usr/local/etc/mpd/mpd.linkup"
        open iface

mpd.links:

sbc1:
        set link type pppoe
        set pppoe iface sis0
        set pppoe service "ISP"
        set pppoe disable incoming
        set pppoe enable originate

sbc2:
        set link type pppoe
        set pppoe iface sis1
        set pppoe service "ISP"
        set pppoe disable incoming
        set pppoe enable originate

mpd.secret:

admin                 adminpass
blah1@sbcglobal.net   pass4blah1
blah2@sbcglobal.net   pass4blah2

mpd.linkup:

#!/bin/sh

# ng0 inet xxx.xxx.xxx.130 192.0.2.100 blah1@sbcglobal.net

echo "${*}" >> /tmp/mpd.linkup

eval $(echo ${1} | sed -e 's/^ng\(.*\)$/export metric="\1"/');

eval $(echo ${3} | sed -e 's/^\([^.]*\.[^.]*\.[^.]*\.\)\(.*\)$/export pub_pref="\1" pub_suf="\2"/')

eval $(echo ${4} | sed -e 's/^\([^.]*\.[^.]*\.[^.]*\.\)\(.*\)$/export her_pref="\1" her_suf="\2"/')

pub_suf=$(expr ${puf_suf} + 1)

idx=0
alias=''
while [ "${idx}" -lt 8 ]
do
        pub_idx=$(expr ${pub_suf} - ${idx})
        pub_addr="${pub_pref}${pub_idx}"
        her_idx=$(expr ${her_suf} + ${idx} + \( ${metric} \* 10 \) + 1)
        her_addr="${her_pref}${her_idx}"
        echo "/sbin/ifconfig ${1} ${alias} ${pub_addr} ${her_addr} netmask 0xffffffff" >> /tmp/mpd.linkup
        /sbin/ifconfig ${1} ${alias} ${pub_addr} ${her_addr} netmask 0xffffffff
        idx=$(expr ${idx} + 1)
        [ -z "${alias}" ] && alias="alias"
done

PacketFilter

int_if="fxp0"
int_net="10.0.0.0/24"
ext1_if="ng0"
ext1_net="xxx.xxx.xxx.137/32"
ext1_gw="192.0.2.101"
ext2_if="ng1"
ext2_net="yyy.yyy.yyy.127/32"
ext2_gw="192.0.2.111"
dnsserv="{ 68.94.156.1/32, 68.94.157.1/32 }"

# Don't simply block packets. This makes debugging rulesets -much- easier.
set block-policy return
#set state-policy if-bound

scrub in all

# AltQ queues
altq on $ext1_if bandwidth 750Kb cbq queue { ack, ssh, dflt, bulk, down }
altq on $ext2_if bandwidth 750Kb cbq queue { ack, ssh, dflt, bulk, down }
queue ack  bandwidth 32Kb  priority 7 cbq(rio, borrow)
queue ssh  bandwidth 128Kb priority 5 cbq(rio, borrow)
queue dflt bandwidth 8Kb   priority 4 cbq(rio, borrow, default)
queue bulk bandwidth 8Kb   priority 2 cbq(rio, borrow)
queue down bandwidth 8Kb   priority 0 cbq(rio, borrow)

# nat/binat: First match wins
# GINKO1
binat on $ext1_if from 10.0.0.2 to any -> xxx.xxx.xxx.135
binat on $ext2_if from 10.0.0.2 to any -> yyy.yyy.yyy.125
# Cobolt
binat on $ext1_if from 10.0.0.3 to any -> xxx.xxx.xxx.136
binat on $ext2_if from 10.0.0.3 to any -> yyy.yyy.yyy.126
# Haverty
binat on $ext1_if from 10.0.0.18 to any -> xxx.xxx.xxx.132
# Riboflavin
binat on $ext1_if from 10.0.0.22 to any -> xxx.xxx.xxx.133
# Potassium
binat on $ext2_if from 10.0.0.23 to any -> yyy.yyy.yyy.131
# NAT rules
nat on $ext1_if from $int_net to any -> $ext1_net
nat on $ext2_if from $int_net to any -> $ext2_net
nat on tun0 from tun0 to any -> $ext1_net

# Forward packets to inner machines
# GINKO1
rdr on $ext1_if from any to xxx.xxx.xxx.135 -> 10.0.0.2
rdr on $ext2_if from any to yyy.yyy.yyy.125 -> 10.0.0.2
# Cobolt
rdr on $ext1_if from any to xxx.xxx.xxx.136 -> 10.0.0.3
rdr on $ext2_if from any to yyy.yyy.yyy.126 -> 10.0.0.3
# Haverty
rdr on $ext1_if from any to xxx.xxx.xxx.132 -> 10.0.0.18
# Riboflavin
rdr on $ext1_if from any to xxx.xxx.xxx.133 -> 10.0.0.22
# Potassium
rdr on $ext2_if from any to yyy.yyy.yyy.131 -> 10.0.0.23
# Tungsten
rdr on $ext1_if proto { tcp, udp } from any to $ext1_net port 40585 -> 10.0.0.8 port 40585
rdr on $ext2_if proto { tcp, udp } from any to $ext2_net port 40585 -> 10.0.0.8 port 42585

# Redirect FTP connections to ftp proxy running on localhost
#rdr on $int_if proto tcp from any to !$int_if port 21 -> 127.0.0.1 port 8021

# pass/block: Last match wins
block in on $ext1_if all
block in on $ext2_if all

# Block SMB/CIFS traffic
block log quick on { $ext1_if, $ext2_if } proto udp from any to any port { 137, 138, 139, 445 }

pass in log quick on $ext1_if from $dnsserv to any keep state queue (ack)
pass in log quick on $ext2_if from $dnsserv to any keep state queue (ack)

# Prioritize VoIP address
pass in on $int_if proto { tcp, udp } from 10.0.0.210 to any keep state tag VOIP
# Deprioritize Tungsten
pass in on $int_if proto { tcp, udp } from 10.0.0.8 to any keep state tag BULK

# Local SSH
pass in on { $ext1_if, $ext2_if } proto tcp from any to { xxx.xxx.xxx.137/32, yyy.yyy.yyy.127/32 } port 22 keep state queue (ssh, ack)

# Load-balance LAN traffic
pass in log on $int_if route-to { (ng0 192.0.2.103), (ng1 192.0.2.115) } round-robin from $int_net to !$int_net flags S/SA keep state
# Load-balance GINKO1
pass in log on $int_if route-to { (ng0 192.0.2.105), (ng1 192.0.2.117) } round-robin from 10.0.0.2 to !$int_net flags S/SA keep state
# Load-balance Cobolt
pass in log on $int_if route-to { (ng0 192.0.2.104), (ng1 192.0.2.116) } round-robin from 10.0.0.3 to !$int_net flags S/SA keep state
# Haverty
pass in log on $int_if route-to (ng0 192.0.2.108) from 10.0.0.18 to !$int_net flags S/SA keep state
# Riboflavin
pass in log on $int_if route-to (ng0 192.0.2.107) from 10.0.0.22 to !$int_net flags S/SA keep state
# Potassium
pass in log on $int_if route-to (ng1 192.0.2.111) from 10.0.0.23 to !$int_net flags S/SA keep state

# Make sure next-hop traffic on NATed gateways goes out the right interface. Not needed for PtP links
#pass in on $int_if route-to { ($ext1_if $ext1_gw) } round-robin from $int_net to $ext1_net keep state
#pass in on $int_if route-to { ($ext2_if $ext2_gw) } round-robin from $int_net to $ext2_net keep state

# Make sure traffic goes out the interface corresponding to its' source address
pass in log on tun0 route-to ($ext1_if $ext1_gw) from $ext1_if to any
pass in log on tun0 route-to ($ext2_if $ext2_gw) from $ext2_if to any
pass out log on $ext1_if route-to ($ext2_if $ext2_gw) from $ext2_if to any
pass out log on $ext2_if route-to ($ext1_if $ext1_gw) from $ext1_if to any
# Block inappropriate traffic. The ISP does this already, but there's no sense wasting bandwidth
#block out log quick on $ext1_if from !$ext1_if to any
#block out log quick on $ext2_if from !$ext2_if to any

# Pass previously NATed connections
# GINKO1
pass in log on { $ext1_if, $ext2_if } from any to 10.0.0.2 keep state queue (dflt, ack)
# Cobolt
pass in log on { $ext1_if, $ext2_if } from any to 10.0.0.3 keep state queue (dflt, ack)
# Haverty
pass in log on $ext1_if from any to 10.0.0.18 keep state queue (dflt, ack)
# Riboflavin
pass in log on $ext1_if from any to 10.0.0.22 keep state queue (dflt, ack)
# Potassium
pass in log on $ext2_if from any to 10.0.0.23 keep state queue (dflt, ack)
# Tungsten
pass in log on { $ext1_if, $ext2_if } proto { tcp, udp } from any to 10.0.0.8 port 40585 keep state queue (bulk, ack)

# FTP Proxy connections
#pass log on $ext1_if proto tcp from any to any user = proxy keep state queue (bulk, ack)
#pass in log on $int_if route-to (lo0 127.0.0.1) proto tcp from any to 127.0.0.1 port 8021 keep state

# Prioritize VoIP
pass out on $ext1_if tagged VOIP keep state queue (ssh, ack)
pass out on $ext2_if tagged VOIP keep state queue (ssh, ack)
# Deprioritize Tungsten
pass out on $ext1_if tagged BULK keep state queue (bulk, ack)
pass out on $ext2_if tagged BULK keep state queue (bulk, ack)

# Bind AIM to a single IP
pass in on $int_if route-to (ng0 192.0.2.103) proto tcp from any to any port 5190 keep state

# Pass all LAN traffic unconditionally, so we don't accidentally lock ourselves out.
pass in on fxp0 from any to fxp0
pass out on fxp0 from fxp0 to any

Injector

A method to reinject packets headed for the default route, so that they can be routed to the proper gateway

  • tun(4) device acts as the default gateway for the machine
  • Any packets it receives must have a source address of one of the public interfaces.
  • Read them, perform a sanity check to make sure their source nor destination address matches that interface, then blindly reinject them
  • Hopefully, this will pass them through the firewall rules again, statelessly, so that the gateway redirection rules can affect them and send them to the proper interface.
  • Starts on tun0. You must ifconfig tun0 up xxx.xxx.xxx.139/32 10.1.1.2 and route add -net default 10.1.1.2 to direct all 'default' packets to this device, so they can be reinjected and hit the gateway redirect rules in pf.
  • tun0 should have a valid source address, as it will be used as the source address for all publicly routed packets coming from the machine itself (DNS lookups, etc..)

tunbounce.c:

#include <errno.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <net/if.h>

int main(int argc, char **argv) {
        static int sockfd = -1;
        static int iffd = -1;
        static char pkt[1500];
        static size_t pkt_len = 0;
        static size_t pkt_len2 = 0;
        static struct ifreq iff;
        static struct ifreq my_iff;
        static int status = -1;

        /* Acquire tunnel device */
        if ((sockfd = open("/dev/tun0", O_RDWR|O_DIRECT)) < 0) {
                perror("open(/dev/tun0)");
                return 32;
        }
        /* Set device up. This is easier than all that mess with sysctls and ioctls */
        if (vfork() == 0) {
                /* Child */
                if (execl("/sbin/ifconfig", "ifconfig", "tun0", "up", (char *)NULL) < 0) {
                        perror("execl(/sbin/ifconfig tun0 up)");
                        _exit(32);
                }
        }
        wait(&status);
        if (WIFEXITED(status) && WEXITSTATUS(status) == 32) {
                printf("Exiting.");
                return 32;
        }
        while (1) {
                if ((pkt_len = read(sockfd, &pkt, 1500)) == -1) {
                        perror("read failed");
                        continue;
                }
                if ((pkt_len2 = write(sockfd, pkt, pkt_len)) == -1) {
                        perror("write failed");
                        continue;
                }else if (pkt_len != pkt_len2) {
                        printf("Partial write: %u of %u\n", pkt_len2, pkt_len);
                        continue;
                }
        }
        return 0;
}

tunbounce.c v0.2: (Doesn't work!)

#include <sys/socket.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <net/if.h>

int main(int argc, char **argv) {
        static int sockfd = -1;
        static int iffd = -1;
        static char pkt[1500];
        static size_t pkt_len = 0;
        static size_t pkt_len2 = 0;
        static struct ifreq iff;
        static struct ifreq my_iff;

        if ((sockfd = open("/dev/tun0", O_RDWR|O_DIRECT)) < 0) {
                perror("open(/dev/tun0)");
                return 32;
        }
        if ((iffd = open("/dev/net/tun0", O_RDWR|O_DIRECT)) < 0) {
                perror("open(/dev/net/tun)");
                return 32;
        }
        if (ioctl(iffd, SIOCGIFFLAGS, &iff) < 0) {
                perror("ioctl (SIOCGIFFLAGS)");
                return 32;
        }
        my_iff.ifr_flags = iff.ifr_flags | IFF_UP;
        if (ioctl(iffd, SIOCSIFFLAGS, my_iff) < 0) {
                perror("ioctl (SIOCSIFFLAGS)");
                return 32;
        }
        printf("%u\n", sockfd);
        while (1) {
                if ((pkt_len = read(sockfd, &pkt, 1500)) == -1) {
                        perror("read failed");
                        continue;
                }
                if ((pkt_len2 = write(sockfd, pkt, pkt_len)) == -1) {
                        perror("write failed");
                        continue;
                }else if (pkt_len != pkt_len2) {
                        printf("Partial write: %u of %u\n", pkt_len2, pkt_len);
                        continue;
                }
        }
        return 0;
}

Script to run the above and make the appropriate changes (cuz doing that in C is way too annoying x.x ) in rcng format dummygw:

#!/bin/sh
#
# PROVIDE: dummygw
# REQUIRE: NETWORKING routing mpd
# KEYWORD: nojail

. /etc/rc.subr

name="dummygw"
start_cmd="dummygw_start"
stop_cmd="dummygw_stop"

dummygw_bin=${tunbounce_bin:-/usr/home/cyberleo/tunbounce}
dummygw_pid=${tunbounce_pid:-/var/run/dummygw.pid}
dummygw_enable=${tunbounce_enable:-NO}
dummygw_addr=${dummygw_addr:-xxx.xxx.xxx.139}
dummygw_peer=${dummygw_peer:-10.44.44.44}

dummygw_checkrunning() {
        cnt=0
        if [ -f "${dummygw_pid}" ]
        then
                pid=$(/bin/cat "${dummygw_pid}")
                cnt=$(/bin/ps -axocommand "${pid:-0}" | /usr/bin/grep -c "$(/usr/bin/basename "${dummygw_bin}")")
        fi
        if [ "${cnt}" -eq 0 ]
        then
                return 1
        fi
        return 0
}

dummygw_start() {
        echo "Establishing bouncing default gateway..."
        if dummygw_checkrunning
        then
                echo "Already running."
                exit 64
        fi
        $dummygw_bin &
        echo ${!} > $dummygw_pid
        sleep 1
        if dummygw_checkrunning
        then
                /sbin/ifconfig tun0 "${dummygw_addr}/32" "${dummygw_peer}"
                /sbin/route delete -net default 2>/dev/null
                /sbin/route add -net default "${dummygw_peer}"
                /sbin/pfctl -Fstate
        else
                echo "Startup failed."
        fi
}
dummygw_stop() {
        echo "Destroying bouncing default gateway..."
        /sbin/route delete -net default 2>/dev/null
        /sbin/route add -net default ${defaultrouter}
        /sbin/ifconfig tun0 down 0.0.0.0 2>/dev/null
        if [ -f "${dummygw_pid}" -a -n "$(cat "${dummygw_pid}")" ]
        then
                kill -TERM $(cat "${dummygw_pid}")
                rm -f "${dummygw_pid}"
        fi
        /sbin/pfctl -Fstate
}

load_rc_config $name
run_rc_command "$1"