Work In Progress!
Create a multi-homed router, based off FreeBSD 6.x
- Support load-balancing NAT outgoing connections across all four routers
- Support proper routing of incoming connections
- No ISP support available (No bundling/multilink, no BGP, etc..)
- PacketFilter preferred
Problems:
- PacketFilter can NAT outgoing connections properly, but incoming connections always use default gateway.
- Statefulness means egress packets don't touch the gateway selection rules in PF
- PF is 'closer' to the device than IPFW, so IPFW only sees untranslated egress packets
Need some way to reinject translated packets, so that they can hit the gateway selection rules.
- All of the above fixed with dummygw (see below)
FTP definitely refuses to work, except for passive FTP on a single-IP binatted private machine.
- Must be bound to a single IP
- Occasional, hard to trace hangs in certain flows (for instance, microsoft.com's homepage)
- Totally doesn't handle single-link failure. Failover script must be crafted.
- DNS works to everywhere except ISP's DNS servers. Why?
Too much traffic causes, almost immediately, 'No buffer space available'
- Restarting uplink subsystem (mpd and dummygw) fixes this for a while.
MPD - Multilink PPP Daemon
- Easy to set up under FreeBSD
- Supports multilink in case ISP gets heads out of asses
- Uses netgraph nodes. All encapsulation done in kernelspace, so it's faster (no context switches)
Must use custom up-script to ensure:
- All six static IPs are added to the node
- None of the added IPs peers are the same (routing error)
- We can specify any arbitrary peer gateway, as it is merely an identifier to pick the appropriate link and doesn't affect the traffic routing.
mpd.conf:
default:
set login admin
load sbcdsl_common
load sbcdsl1
load sbcdsl2
sbcdsl_common:
set iface disable on-demand
set iface idle 0
set bundle enable multilink
set link no acfcomp protocomp
set link disable pap chap
set link accept chap
set link mtu 1492
set link keep-alive 10 60
set link ipcp yes vjcomp
set ipcp ranges 0.0.0.0/0 0.0.0.0/0
set iface up-script /usr/local/etc/mpd/mpd.linkup
sbcdsl1:
new -i ng0 sbcdsl1 sbc1
set iface addrs 10.0.0.1 10.0.0.2
set bundle authname "blah1@sbcglobal.net"
set iface up-script "/usr/local/etc/mpd/mpd.linkup"
open iface
sbcdsl2:
new -i ng1 sbcdsl2 sbc2
set iface addrs 10.0.0.3 10.0.0.4
set bundle authname "blah2@sbcglobal.net"
set iface up-script "/usr/local/etc/mpd/mpd.linkup"
open iface
mpd.links:
sbc1:
set link type pppoe
set pppoe iface sis0
set pppoe service "ISP"
set pppoe disable incoming
set pppoe enable originate
sbc2:
set link type pppoe
set pppoe iface sis1
set pppoe service "ISP"
set pppoe disable incoming
set pppoe enable originate
mpd.secret:
admin adminpass blah1@sbcglobal.net pass4blah1 blah2@sbcglobal.net pass4blah2
mpd.linkup:
#!/bin/sh
# ng0 inet xxx.xxx.xxx.130 192.0.2.100 blah1@sbcglobal.net
echo "${*}" >> /tmp/mpd.linkup
eval $(echo ${1} | sed -e 's/^ng\(.*\)$/export metric="\1"/');
eval $(echo ${3} | sed -e 's/^\([^.]*\.[^.]*\.[^.]*\.\)\(.*\)$/export pub_pref="\1" pub_suf="\2"/')
eval $(echo ${4} | sed -e 's/^\([^.]*\.[^.]*\.[^.]*\.\)\(.*\)$/export her_pref="\1" her_suf="\2"/')
pub_suf=$(expr ${puf_suf} + 1)
idx=0
alias=''
while [ "${idx}" -lt 8 ]
do
pub_idx=$(expr ${pub_suf} - ${idx})
pub_addr="${pub_pref}${pub_idx}"
her_idx=$(expr ${her_suf} + ${idx} + \( ${metric} \* 10 \) + 1)
her_addr="${her_pref}${her_idx}"
echo "/sbin/ifconfig ${1} ${alias} ${pub_addr} ${her_addr} netmask 0xffffffff" >> /tmp/mpd.linkup
/sbin/ifconfig ${1} ${alias} ${pub_addr} ${her_addr} netmask 0xffffffff
idx=$(expr ${idx} + 1)
[ -z "${alias}" ] && alias="alias"
done
PacketFilter
int_if="fxp0"
int_net="10.0.0.0/24"
ext1_if="ng0"
ext1_net="xxx.xxx.xxx.137/32"
ext1_gw="192.0.2.101"
ext2_if="ng1"
ext2_net="yyy.yyy.yyy.127/32"
ext2_gw="192.0.2.111"
dnsserv="{ 68.94.156.1/32, 68.94.157.1/32 }"
# Don't simply block packets. This makes debugging rulesets -much- easier.
set block-policy return
#set state-policy if-bound
scrub in all
# AltQ queues
altq on $ext1_if bandwidth 750Kb cbq queue { ack, ssh, dflt, bulk, down }
altq on $ext2_if bandwidth 750Kb cbq queue { ack, ssh, dflt, bulk, down }
queue ack bandwidth 32Kb priority 7 cbq(rio, borrow)
queue ssh bandwidth 128Kb priority 5 cbq(rio, borrow)
queue dflt bandwidth 8Kb priority 4 cbq(rio, borrow, default)
queue bulk bandwidth 8Kb priority 2 cbq(rio, borrow)
queue down bandwidth 8Kb priority 0 cbq(rio, borrow)
# nat/binat: First match wins
# GINKO1
binat on $ext1_if from 10.0.0.2 to any -> xxx.xxx.xxx.135
binat on $ext2_if from 10.0.0.2 to any -> yyy.yyy.yyy.125
# Cobolt
binat on $ext1_if from 10.0.0.3 to any -> xxx.xxx.xxx.136
binat on $ext2_if from 10.0.0.3 to any -> yyy.yyy.yyy.126
# Haverty
binat on $ext1_if from 10.0.0.18 to any -> xxx.xxx.xxx.132
# Riboflavin
binat on $ext1_if from 10.0.0.22 to any -> xxx.xxx.xxx.133
# Potassium
binat on $ext2_if from 10.0.0.23 to any -> yyy.yyy.yyy.131
# NAT rules
nat on $ext1_if from $int_net to any -> $ext1_net
nat on $ext2_if from $int_net to any -> $ext2_net
nat on tun0 from tun0 to any -> $ext1_net
# Forward packets to inner machines
# GINKO1
rdr on $ext1_if from any to xxx.xxx.xxx.135 -> 10.0.0.2
rdr on $ext2_if from any to yyy.yyy.yyy.125 -> 10.0.0.2
# Cobolt
rdr on $ext1_if from any to xxx.xxx.xxx.136 -> 10.0.0.3
rdr on $ext2_if from any to yyy.yyy.yyy.126 -> 10.0.0.3
# Haverty
rdr on $ext1_if from any to xxx.xxx.xxx.132 -> 10.0.0.18
# Riboflavin
rdr on $ext1_if from any to xxx.xxx.xxx.133 -> 10.0.0.22
# Potassium
rdr on $ext2_if from any to yyy.yyy.yyy.131 -> 10.0.0.23
# Tungsten
rdr on $ext1_if proto { tcp, udp } from any to $ext1_net port 40585 -> 10.0.0.8 port 40585
rdr on $ext2_if proto { tcp, udp } from any to $ext2_net port 40585 -> 10.0.0.8 port 42585
# Redirect FTP connections to ftp proxy running on localhost
#rdr on $int_if proto tcp from any to !$int_if port 21 -> 127.0.0.1 port 8021
# pass/block: Last match wins
block in on $ext1_if all
block in on $ext2_if all
# Block SMB/CIFS traffic
block log quick on { $ext1_if, $ext2_if } proto udp from any to any port { 137, 138, 139, 445 }
pass in log quick on $ext1_if from $dnsserv to any keep state queue (ack)
pass in log quick on $ext2_if from $dnsserv to any keep state queue (ack)
# Prioritize VoIP address
pass in on $int_if proto { tcp, udp } from 10.0.0.210 to any keep state tag VOIP
# Deprioritize Tungsten
pass in on $int_if proto { tcp, udp } from 10.0.0.8 to any keep state tag BULK
# Local SSH
pass in on { $ext1_if, $ext2_if } proto tcp from any to { xxx.xxx.xxx.137/32, yyy.yyy.yyy.127/32 } port 22 keep state queue (ssh, ack)
# Load-balance LAN traffic
pass in log on $int_if route-to { (ng0 192.0.2.103), (ng1 192.0.2.115) } round-robin from $int_net to !$int_net flags S/SA keep state
# Load-balance GINKO1
pass in log on $int_if route-to { (ng0 192.0.2.105), (ng1 192.0.2.117) } round-robin from 10.0.0.2 to !$int_net flags S/SA keep state
# Load-balance Cobolt
pass in log on $int_if route-to { (ng0 192.0.2.104), (ng1 192.0.2.116) } round-robin from 10.0.0.3 to !$int_net flags S/SA keep state
# Haverty
pass in log on $int_if route-to (ng0 192.0.2.108) from 10.0.0.18 to !$int_net flags S/SA keep state
# Riboflavin
pass in log on $int_if route-to (ng0 192.0.2.107) from 10.0.0.22 to !$int_net flags S/SA keep state
# Potassium
pass in log on $int_if route-to (ng1 192.0.2.111) from 10.0.0.23 to !$int_net flags S/SA keep state
# Make sure next-hop traffic on NATed gateways goes out the right interface. Not needed for PtP links
#pass in on $int_if route-to { ($ext1_if $ext1_gw) } round-robin from $int_net to $ext1_net keep state
#pass in on $int_if route-to { ($ext2_if $ext2_gw) } round-robin from $int_net to $ext2_net keep state
# Make sure traffic goes out the interface corresponding to its' source address
pass in log on tun0 route-to ($ext1_if $ext1_gw) from $ext1_if to any
pass in log on tun0 route-to ($ext2_if $ext2_gw) from $ext2_if to any
pass out log on $ext1_if route-to ($ext2_if $ext2_gw) from $ext2_if to any
pass out log on $ext2_if route-to ($ext1_if $ext1_gw) from $ext1_if to any
# Block inappropriate traffic. The ISP does this already, but there's no sense wasting bandwidth
#block out log quick on $ext1_if from !$ext1_if to any
#block out log quick on $ext2_if from !$ext2_if to any
# Pass previously NATed connections
# GINKO1
pass in log on { $ext1_if, $ext2_if } from any to 10.0.0.2 keep state queue (dflt, ack)
# Cobolt
pass in log on { $ext1_if, $ext2_if } from any to 10.0.0.3 keep state queue (dflt, ack)
# Haverty
pass in log on $ext1_if from any to 10.0.0.18 keep state queue (dflt, ack)
# Riboflavin
pass in log on $ext1_if from any to 10.0.0.22 keep state queue (dflt, ack)
# Potassium
pass in log on $ext2_if from any to 10.0.0.23 keep state queue (dflt, ack)
# Tungsten
pass in log on { $ext1_if, $ext2_if } proto { tcp, udp } from any to 10.0.0.8 port 40585 keep state queue (bulk, ack)
# FTP Proxy connections
#pass log on $ext1_if proto tcp from any to any user = proxy keep state queue (bulk, ack)
#pass in log on $int_if route-to (lo0 127.0.0.1) proto tcp from any to 127.0.0.1 port 8021 keep state
# Prioritize VoIP
pass out on $ext1_if tagged VOIP keep state queue (ssh, ack)
pass out on $ext2_if tagged VOIP keep state queue (ssh, ack)
# Deprioritize Tungsten
pass out on $ext1_if tagged BULK keep state queue (bulk, ack)
pass out on $ext2_if tagged BULK keep state queue (bulk, ack)
# Bind AIM to a single IP
pass in on $int_if route-to (ng0 192.0.2.103) proto tcp from any to any port 5190 keep state
# Pass all LAN traffic unconditionally, so we don't accidentally lock ourselves out.
pass in on fxp0 from any to fxp0
pass out on fxp0 from fxp0 to any
Injector
A method to reinject packets headed for the default route, so that they can be routed to the proper gateway
- tun(4) device acts as the default gateway for the machine
- Any packets it receives must have a source address of one of the public interfaces.
- Read them, perform a sanity check to make sure their source nor destination address matches that interface, then blindly reinject them
- Hopefully, this will pass them through the firewall rules again, statelessly, so that the gateway redirection rules can affect them and send them to the proper interface.
- Starts on tun0. You must ifconfig tun0 up xxx.xxx.xxx.139/32 10.1.1.2 and route add -net default 10.1.1.2 to direct all 'default' packets to this device, so they can be reinjected and hit the gateway redirect rules in pf.
- tun0 should have a valid source address, as it will be used as the source address for all publicly routed packets coming from the machine itself (DNS lookups, etc..)
tunbounce.c:
#include <errno.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <net/if.h>
int main(int argc, char **argv) {
static int sockfd = -1;
static int iffd = -1;
static char pkt[1500];
static size_t pkt_len = 0;
static size_t pkt_len2 = 0;
static struct ifreq iff;
static struct ifreq my_iff;
static int status = -1;
/* Acquire tunnel device */
if ((sockfd = open("/dev/tun0", O_RDWR|O_DIRECT)) < 0) {
perror("open(/dev/tun0)");
return 32;
}
/* Set device up. This is easier than all that mess with sysctls and ioctls */
if (vfork() == 0) {
/* Child */
if (execl("/sbin/ifconfig", "ifconfig", "tun0", "up", (char *)NULL) < 0) {
perror("execl(/sbin/ifconfig tun0 up)");
_exit(32);
}
}
wait(&status);
if (WIFEXITED(status) && WEXITSTATUS(status) == 32) {
printf("Exiting.");
return 32;
}
while (1) {
if ((pkt_len = read(sockfd, &pkt, 1500)) == -1) {
perror("read failed");
continue;
}
if ((pkt_len2 = write(sockfd, pkt, pkt_len)) == -1) {
perror("write failed");
continue;
}else if (pkt_len != pkt_len2) {
printf("Partial write: %u of %u\n", pkt_len2, pkt_len);
continue;
}
}
return 0;
}
tunbounce.c v0.2: (Doesn't work!)
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <net/if.h>
int main(int argc, char **argv) {
static int sockfd = -1;
static int iffd = -1;
static char pkt[1500];
static size_t pkt_len = 0;
static size_t pkt_len2 = 0;
static struct ifreq iff;
static struct ifreq my_iff;
if ((sockfd = open("/dev/tun0", O_RDWR|O_DIRECT)) < 0) {
perror("open(/dev/tun0)");
return 32;
}
if ((iffd = open("/dev/net/tun0", O_RDWR|O_DIRECT)) < 0) {
perror("open(/dev/net/tun)");
return 32;
}
if (ioctl(iffd, SIOCGIFFLAGS, &iff) < 0) {
perror("ioctl (SIOCGIFFLAGS)");
return 32;
}
my_iff.ifr_flags = iff.ifr_flags | IFF_UP;
if (ioctl(iffd, SIOCSIFFLAGS, my_iff) < 0) {
perror("ioctl (SIOCSIFFLAGS)");
return 32;
}
printf("%u\n", sockfd);
while (1) {
if ((pkt_len = read(sockfd, &pkt, 1500)) == -1) {
perror("read failed");
continue;
}
if ((pkt_len2 = write(sockfd, pkt, pkt_len)) == -1) {
perror("write failed");
continue;
}else if (pkt_len != pkt_len2) {
printf("Partial write: %u of %u\n", pkt_len2, pkt_len);
continue;
}
}
return 0;
}
Script to run the above and make the appropriate changes (cuz doing that in C is way too annoying x.x ) in rcng format dummygw:
#!/bin/sh
#
# PROVIDE: dummygw
# REQUIRE: NETWORKING routing mpd
# KEYWORD: nojail
. /etc/rc.subr
name="dummygw"
start_cmd="dummygw_start"
stop_cmd="dummygw_stop"
dummygw_bin=${tunbounce_bin:-/usr/home/cyberleo/tunbounce}
dummygw_pid=${tunbounce_pid:-/var/run/dummygw.pid}
dummygw_enable=${tunbounce_enable:-NO}
dummygw_addr=${dummygw_addr:-xxx.xxx.xxx.139}
dummygw_peer=${dummygw_peer:-10.44.44.44}
dummygw_checkrunning() {
cnt=0
if [ -f "${dummygw_pid}" ]
then
pid=$(/bin/cat "${dummygw_pid}")
cnt=$(/bin/ps -axocommand "${pid:-0}" | /usr/bin/grep -c "$(/usr/bin/basename "${dummygw_bin}")")
fi
if [ "${cnt}" -eq 0 ]
then
return 1
fi
return 0
}
dummygw_start() {
echo "Establishing bouncing default gateway..."
if dummygw_checkrunning
then
echo "Already running."
exit 64
fi
$dummygw_bin &
echo ${!} > $dummygw_pid
sleep 1
if dummygw_checkrunning
then
/sbin/ifconfig tun0 "${dummygw_addr}/32" "${dummygw_peer}"
/sbin/route delete -net default 2>/dev/null
/sbin/route add -net default "${dummygw_peer}"
/sbin/pfctl -Fstate
else
echo "Startup failed."
fi
}
dummygw_stop() {
echo "Destroying bouncing default gateway..."
/sbin/route delete -net default 2>/dev/null
/sbin/route add -net default ${defaultrouter}
/sbin/ifconfig tun0 down 0.0.0.0 2>/dev/null
if [ -f "${dummygw_pid}" -a -n "$(cat "${dummygw_pid}")" ]
then
kill -TERM $(cat "${dummygw_pid}")
rm -f "${dummygw_pid}"
fi
/sbin/pfctl -Fstate
}
load_rc_config $name
run_rc_command "$1"