CDNWiki - View Source: KnowledgeBase/FreeBSD/MultiHomedRouter

Note: You are viewing an old version of this page. View the current version.
!!!Work In Progress!

!!!Create a multi-homed router, based off FreeBSD 6.x
* Support load-balancing NAT outgoing connections across all four routers
* Support proper routing of incoming connections
* No ISP support available (No bundling/multilink, no BGP, etc..)
* ~PacketFilter preferred

!Problems:
* ~PacketFilter can NAT outgoing connections properly, but incoming connections always use default gateway.
  * Fixed with dummygw (see below)
* Statefulness means egress packets don't touch the gateway selection rules in PF
  * Fixed with dummygw (see below)
* PF is 'closer' to the device than IPFW, so IPFW only sees untranslated egress packets
* Need some way to reinject translated packets, so that they can hit the gateway selection rules.
  * Fixed with dummygw (see below)

!MPD - Multilink PPP Daemon
* Easy to set up under FreeBSD
* Supports multilink in case ISP gets heads out of asses
* Uses netgraph nodes. All encapsulation done in kernelspace, so it's faster (no context switches)
* Must use custom up-script to ensure:
  * All six static IPs are added to the node
  * None of the added IPs peers are the same (routing error)
  * We can specify any arbitrary peer gateway, as it is merely an identifier to pick the appropriate link and doesn't affect the traffic routing.

mpd.conf:
<verbatim>
default:
        set login admin
        load sbcdsl_common
        load sbcdsl1
        load sbcdsl2
sbcdsl_common:
        set iface disable on-demand
        set iface idle 0
        set bundle enable multilink
        set link no acfcomp protocomp
        set link disable pap chap
        set link accept chap
        set link mtu 1492
        set link keep-alive 10 60
        set link ipcp yes vjcomp
        set ipcp ranges 0.0.0.0/0 0.0.0.0/0
        set iface up-script /usr/local/etc/mpd/mpd.linkup
sbcdsl1:
        new -i ng0 sbcdsl1 sbc1
        set iface addrs 10.0.0.1 10.0.0.2
        set bundle authname "blah1@sbcglobal.net"
        set iface up-script "/usr/local/etc/mpd/mpd.linkup"
        open iface
sbcdsl2:
        new -i ng1 sbcdsl2 sbc2
        set iface addrs 10.0.0.3 10.0.0.4
        set bundle authname "blah2@sbcglobal.net"
        set iface up-script "/usr/local/etc/mpd/mpd.linkup"
        open iface
</verbatim>

mpd.links:
<verbatim>
sbc1:
        set link type pppoe
        set pppoe iface sis0
        set pppoe service "ISP"
        set pppoe disable incoming
        set pppoe enable originate

sbc2:
        set link type pppoe
        set pppoe iface sis1
        set pppoe service "ISP"
        set pppoe disable incoming
        set pppoe enable originate
</verbatim>

mpd.secret:
<verbatim>
admin                 adminpass
blah1@sbcglobal.net   pass4blah1
blah2@sbcglobal.net   pass4blah2
</verbatim>

mpd.linkup:
<verbatim>
#!/bin/sh

# ng0 inet xxx.xxx.xxx.130 192.0.2.100 blah1@sbcglobal.net

echo "${*}" >> /tmp/mpd.linkup

eval $(echo ${1} | sed -e 's/^ng\(.*\)$/export metric="\1"/');

eval $(echo ${3} | sed -e 's/^\([^.]*\.[^.]*\.[^.]*\.\)\(.*\)$/export pub_pref="\1" pub_suf="\2"/')

eval $(echo ${4} | sed -e 's/^\([^.]*\.[^.]*\.[^.]*\.\)\(.*\)$/export her_pref="\1" her_suf="\2"/')

pub_suf=$(expr ${puf_suf} - 1)

idx=0
alias=''
while [ "${idx}" -lt 8 ]
do
        pub_idx=$(expr ${pub_suf} - ${idx})
        pub_addr="${pub_pref}${pub_idx}"
        her_idx=$(expr ${her_suf} + ${idx} + \( ${metric} \* 10 \) + 1)
        her_addr="${her_pref}${her_idx}"
        echo "/sbin/ifconfig ${1} ${alias} ${pub_addr} ${her_addr} netmask 0xffffffff" >> /tmp/mpd.linkup
        /sbin/ifconfig ${1} ${alias} ${pub_addr} ${her_addr} netmask 0xffffffff
        idx=$(expr ${idx} + 1)
        [ -z "${alias}" ] && alias="alias"
done
</verbatim>

!~PacketFilter
<verbatim>
int_if="fxp0"
int_net="10.0.0.0/24"
ext1_if="ng0"
ext1_net="xxx.xxx.xxx.138/32"
ext1_gw="192.0.2.101"
ext2_if="ng1"
ext2_net="yyy.yyy.yyy.130/32"
ext2_gw="192.0.2.102"

set block-policy return

scrub in all
set block-policy return

scrub in all

altq on $ext1_if bandwidth 750Kb cbq queue { ack, ssh, dflt, bulk, down }
altq on $ext2_if bandwidth 750Kb cbq queue { ack, ssh, dflt, bulk, down }
queue ack  bandwidth 32Kb  priority 7 cbq(rio, borrow)
queue ssh  bandwidth 128Kb priority 5 cbq(rio, borrow)
queue dflt bandwidth 8Kb   priority 4 cbq(rio, borrow, default)
queue bulk bandwidth 8Kb   priority 2 cbq(rio, borrow)
queue down bandwidth 8Kb   priority 0 cbq(rio, borrow)

nat on $ext1_if from $int_net to any -> ($ext1_if)
nat on $ext2_if from $int_net to any -> ($ext2_if)
# Packets from this box will be generated with the source address of the default gateway. NAT them to their proper home.
nat on tun0 from tun0 to any -> ($ext1_if)

# Some test redirects
rdr on $ext1_if proto tcp from any to xxx.xxx.xxx.133 port 22 -> 10.0.0.18 port 22
rdr on $ext2_if proto tcp from any to yyy.yyy.yyy.125 port 22 -> 10.0.0.18 port 22

#pass in log on $ext1_if keep state
#pass in log on $ext2_if keep state

# Load-balance LAN traffic
pass in log on $int_if route-to { ($ext1_if $ext1_gw), ($ext2_if $ext2_gw) } round-robin from $int_net to !$int_if flags S/SA keep state
# Don't load-balance traffic that's one-hop away from the public interfaces. Not useful for DSL/PPPoE.
#pass in on $int_if route-to { ($ext1_if $ext1_gw) } round-robin from $int_net to $ext1_net keep state
#pass in on $int_if route-to { ($ext2_if $ext2_gw) } round-robin from $int_net to $ext2_net keep state

# Make sure traffic goes where it's needed
pass in log on tun0 route-to ($ext1_if $ext1_gw) from $ext1_if to any
pass in log on tun0 route-to ($ext2_if $ext2_gw) from $ext2_if to any
pass out log on $ext1_if route-to ($ext2_if $ext2_gw) from $ext2_if to any
pass out log on $ext2_if route-to ($ext1_if $ext1_gw) from $ext1_if to any
# Block inappropriate traffic. The ISP does this anyways.
#block out quick on $ext1_if from !$ext1_if to any
#block out quick on $ext2_if from !$ext2_if to any

# Pass/queue rules for the previous rdr rules
pass in on $ext1_if proto tcp from any to 10.0.0.18 port 22 keep state queue (ssh, ack)
pass in on $ext2_if proto tcp from any to 10.0.0.18 port 22 keep state queue (ssh, ack)

# Anti-lockouts, cuz I'm doing this remotely. >@.@<
pass in on fxp0 from any to fxp0
pass out on fxp0 from fxp0 to any
</verbatim>

!Injector
A method to reinject packets headed for the default route, so that they can be routed to the proper gateway
* tun(4) device acts as the default gateway for the machine
* Any packets it receives must have a source address of one of the public interfaces.
* Read them, perform a sanity check to make sure their source nor destination address matches that interface, then blindly reinject them
* Hopefully, this will pass them through the firewall rules again, statelessly, so that the gateway redirection rules can affect them and send them to the proper interface.
* Starts on tun0. You must ifconfig tun0 up xxx.xxx.xxx.139/32 10.1.1.2 and route add -net default 10.1.1.2 to direct all 'default' packets to this device, so they can be reinjected and hit the gateway redirect rules in pf.
* tun0 should have a valid source address, as it will be used as the source address for all publicly routed packets coming from the machine itself (DNS lookups, etc..)
tunbounce.c:
<verbatim>
#include <errno.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <net/if.h>

int main(int argc, char **argv) {
        static int sockfd = -1;
        static int iffd = -1;
        static char pkt[1500];
        static size_t pkt_len = 0;
        static size_t pkt_len2 = 0;
        static struct ifreq iff;
        static struct ifreq my_iff;
        static int status = -1;

        /* Acquire tunnel device */
        if ((sockfd = open("/dev/tun0", O_RDWR|O_DIRECT)) < 0) {
                perror("open(/dev/tun0)");
                return 32;
        }
        /* Set device up. This is easier than all that mess with sysctls and ioctls */
        if (vfork() == 0) {
                /* Child */
                if (execl("/sbin/ifconfig", "ifconfig", "tun0", "up", (char *)NULL) < 0) {
                        perror("execl(/sbin/ifconfig tun0 up)");
                        _exit(32);
                }
        }
        wait(&status);
        if (WIFEXITED(status) && WEXITSTATUS(status) == 32) {
                printf("Exiting.");
                return 32;
        }
        while (1) {
                if ((pkt_len = read(sockfd, &pkt, 1500)) == -1) {
                        perror("read failed");
                        continue;
                }
                if ((pkt_len2 = write(sockfd, pkt, pkt_len)) == -1) {
                        perror("write failed");
                        continue;
                }else if (pkt_len != pkt_len2) {
                        printf("Partial write: %u of %u\n", pkt_len2, pkt_len);
                        continue;
                }
        }
        return 0;
}
</verbatim>

tunbounce.c v0.2: (*Doesn't work!*)
<verbatim>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <net/if.h>

int main(int argc, char **argv) {
        static int sockfd = -1;
        static int iffd = -1;
        static char pkt[1500];
        static size_t pkt_len = 0;
        static size_t pkt_len2 = 0;
        static struct ifreq iff;
        static struct ifreq my_iff;

        if ((sockfd = open("/dev/tun0", O_RDWR|O_DIRECT)) < 0) {
                perror("open(/dev/tun0)");
                return 32;
        }
        if ((iffd = open("/dev/net/tun0", O_RDWR|O_DIRECT)) < 0) {
                perror("open(/dev/net/tun)");
                return 32;
        }
        if (ioctl(iffd, SIOCGIFFLAGS, &iff) < 0) {
                perror("ioctl (SIOCGIFFLAGS)");
                return 32;
        }
        my_iff.ifr_flags = iff.ifr_flags | IFF_UP;
        if (ioctl(iffd, SIOCSIFFLAGS, my_iff) < 0) {
                perror("ioctl (SIOCSIFFLAGS)");
                return 32;
        }
        printf("%u\n", sockfd);
        while (1) {
                if ((pkt_len = read(sockfd, &pkt, 1500)) == -1) {
                        perror("read failed");
                        continue;
                }
                if ((pkt_len2 = write(sockfd, pkt, pkt_len)) == -1) {
                        perror("write failed");
                        continue;
                }else if (pkt_len != pkt_len2) {
                        printf("Partial write: %u of %u\n", pkt_len2, pkt_len);
                        continue;
                }
        }
        return 0;
}
</verbatim>

Script to run the above and make the appropriate changes (cuz doing that in C is way too annoying =x.x= ) in rcng format
dummygw:
<verbatim>
#!/bin/sh
#
# PROVIDE: dummygw
# REQUIRE: NETWORKING routing mpd
# KEYWORD: nojail

. /etc/rc.subr

name="dummygw"
start_cmd="dummygw_start"
stop_cmd="dummygw_stop"

dummygw_bin=${tunbounce_bin:-/usr/home/cyberleo/tunbounce}
dummygw_pid=${tunbounce_pid:-/var/run/dummygw.pid}
dummygw_enable=${tunbounce_enable:-NO}
dummygw_addr=${dummygw_addr:-xxx.xxx.xxx.139}
dummygw_peer=${dummygw_peer:-10.44.44.44}

dummygw_checkrunning() {
        cnt=0
        if [ -f "${dummygw_pid}" ]
        then
                pid=$(/bin/cat "${dummygw_pid}")
                cnt=$(/bin/ps -axocommand "${pid:-0}" | /usr/bin/grep -c "$(/usr/bin/basename "${dummygw_bin}")")
        fi
        if [ "${cnt}" -eq 0 ]
        then
                return 1
        fi
        return 0
}

dummygw_start() {
        echo "Establishing bouncing default gateway..."
        if dummygw_checkrunning
        then
                echo "Already running."
                exit 64
        fi
        $dummygw_bin &
        echo ${!} > $dummygw_pid
        sleep 1
        if dummygw_checkrunning
        then
                /sbin/ifconfig tun0 "${dummygw_addr}/32" "${dummygw_peer}"
                /sbin/route delete -net default 2>/dev/null
                /sbin/route add -net default "${dummygw_peer}"
                /sbin/pfctl -Fstate
        else
                echo "Startup failed."
        fi
}
dummygw_stop() {
        echo "Destroying bouncing default gateway..."
        /sbin/route delete -net default 2>/dev/null
        /sbin/route add -net default ${defaultrouter}
        /sbin/ifconfig tun0 down 0.0.0.0 2>/dev/null
        if [ -f "${dummygw_pid}" -a -n "$(cat "${dummygw_pid}")" ]
        then
                kill -TERM $(cat "${dummygw_pid}")
                rm -f "${dummygw_pid}"
        fi
        /sbin/pfctl -Fstate
}

load_rc_config $name
run_rc_command "$1"
</verbatim>