diff --git a/.github/workflows/build-btf-kernel.yml b/.github/workflows/build-btf-kernel.yml index 833d907..24fe700 100644 --- a/.github/workflows/build-btf-kernel.yml +++ b/.github/workflows/build-btf-kernel.yml @@ -34,10 +34,10 @@ jobs: - boardfamily: sunxi branch: legacy representative: orangepi-r1 - # TODO Enable for R1plus and others as needed - # - boardfamily: rockchip64 - # branch: current - # representative: rockpro64 + # TODO Enable for others as needed + - boardfamily: rockchip64 + branch: legacy + representative: orangepi-r1plus fail-fast: false env: BOARD_FAMILY: ${{ matrix.boardfamily }} diff --git a/Makefile b/Makefile index 3064f2c..cbb7a43 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ run-root: all: GOARCH=arm GOOS=linux go build -o build/nethadone-arm-linux ./cmd/nethadone.go GOARCH=arm64 GOOS=linux go build -o build/nethadone-arm64-linux ./cmd/nethadone.go - GOARCH=amd64 GOOS=linux go build -o build/nethadone-arm-linux ./cmd/nethadone.go + GOARCH=amd64 GOOS=linux go build -o build/nethadone-amd64-linux ./cmd/nethadone.go build: go build ./cmd/nethadone.go diff --git a/README.md b/README.md index 4816d5b..388ef1f 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ etc.) while avoiding the downsides of excessive use. Nethadone is in active development and has been tested on the following devices: -* [Orange Pi R1 Plus](http://www.orangepi.org/orangepiwiki/index.php/Orange_Pi_R1_Plus) -* [Orange Pi R1](http://www.orangepi.org/orangepiwiki/index.php/Orange_Pi_R1) +* [Orange Pi R1 Plus](http://www.orangepi.org/html/hardWare/computerAndMicrocontrollers/details/orange-pi-R1-Plus.html) +* [Orange Pi R1](http://www.orangepi.org/html/hardWare/computerAndMicrocontrollers/details/Orange-Pi-R1.html) +* [Orange Pi Zero](http://www.orangepi.org/html/hardWare/computerAndMicrocontrollers/details/Orange-Pi-Zero-LTS.html) -Any similar device using an armv7 or arm64 chipset should work. -Please check out [doc/install.md](doc/install.md) for details. +Please check out [doc/install.md](doc/install.md) for more details. For development or testing on a local VM, please see [doc/development.md](doc/development.md) diff --git a/cmd/nethadone.go b/cmd/nethadone.go index 31102a3..6b7b867 100644 --- a/cmd/nethadone.go +++ b/cmd/nethadone.go @@ -19,16 +19,20 @@ import ( func main() { - wanIf := flag.String("wan-interface", "eth0", "Interface connecting out to internet") - lanIf := flag.String("lan-interface", "eth1", "Interface connected to local network") + wanIf := flag.String("wan-interface", "", "Interface connecting out to internet") + lanIf := flag.String("lan-interface", "", "Interface connected to local network") configFile := flag.String("config-file", "nethadone.yml", "Configuration file") flag.Parse() config.ParseConfig(*configFile) // Command line parameters override anything that might be in the config - config.Cfg.LanInterface = *lanIf - config.Cfg.WanInterface = *wanIf + if *lanIf != "" { + config.Cfg.LanInterface = *lanIf + } + if *wanIf != "" { + config.Cfg.WanInterface = *wanIf + } log.Println("Configuration: ", repr.String(config.Cfg, repr.Indent(" "))) database.Connect() diff --git a/config/nethadone.yml b/config/nethadone.yml index 8ed723e..e6c44cb 100644 --- a/config/nethadone.yml +++ b/config/nethadone.yml @@ -1,7 +1,7 @@ # The WAN interface should have access to the internet wan_interface: "eth0" # The LAN interface is where your local devices are -lan_nterface: "eth1" +lan_interface: "eth1" cfg_db: "/root/cfg.db" dns_db: "/root/dns.db" diff --git a/doc/install.md b/doc/install.md index fe6f2da..f4e853e 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,101 +1,109 @@ # Installation -Nethadone still, unfortunately, requires a lot of manual steps to -get working. -It is a priority to make this installation process less cumbersome, -but anyone brave enough to attempt this in its current state -has my infinite gratitude! +## Requirements -## Requirements +Nethadone has been tested on the latest Ubuntu Noble-based versions of +Armbian (24.04 or 24.08). You can download an image for your board at +the [Armbian download page](https://www.armbian.com/download/?tx_maker=xunlong). -### Hardware -Nethadone has been tested on: +## Image prep -* VMWare-based VM (for development) -* Orange Pi R1Plus / R1Plus LTS - * 64-bit Rockchip RK3328 - * 1GB RAM - * 2x GBit ethernet - * Wifi (not functional w/ Armbian yet) -* Orange Pi R1 - * 32-bit Allwinner H3 - * 2 x 100Mbit ethernet - * Wifi (working) - * 256MB / 512MB RAM +The contents of `scripts/customize-image.sh` will need to be run, +as root, in a fresh installation of the image you downloaded. +If you want to save time, you can prep using the chroot build +script, eg: -In theory, any device with two network interfaces should work, but I have not had a chance to test others. +```bash +cd scripts +./build.sh -i /tmp/Armbian_community_24.8.0-trunk.554_Orangepizero_noble_current_6.6.43.img -m ~/mnt -a arm +``` -An SD card 16GB or greater is recommended. +`build.sh` makes liberal use of `sudo` so you may want to run this +on a VM or cloud server with a fast link. -### OS +Replace the `-i` and `-m` options with wherever you uncompressed the +Armbian image above, and an empty folder to use for a mount point. +The `-a` option indicates whether you are building for an arm64 +or armv7 image. + +> [!TIP] +> `build.sh` is just a chroot wrapper that pre-installs everything +required for nethadone to run. If you prefer to do the installation +of required packages directly on the SBC, you can copy paste from +`customize-image.sh` after you have flashed the stock image using +[Balena etcher](https://etcher.balena.io/) or whichever flashing tool you prefer. -Ubuntu-based 22.04 LTS or derivative (eg. Armbian 23.8) -Ubuntu 22.04 or 24.04 LTS and derivatives (i.e. Armbian) are working -Other OS versions may work, but the further away from Linux -6.1 (either newer or older), the more likely eBPF -issues may be encountered. +At a high level, the `customize-image.sh` script: -## OS Setup - Orange Pi R1+ (Arm64) +* Installs and configures compiler dependencies, bpftool and prometheus +* Creates a systemd service for nethadone to run on boot +* Downloads and installs a BTF-enabled kernel +* Downloads the nethadone binary from Github -> [!NOTE] -> I am in the process of streamlining this installation process, -> first for the armv7 Orange Pi R1 and eventually for the R1+. -### Base image -For the Orange Pi R1+, you need to get the Armbian 23.8.1 Jammy build: +## Nethadone post-installation -https://xogium.performanceservers.nl/archive/orangepi-r1plus/archive/Armbian_23.8.1_Orangepi-r1plus_jammy_current_6.1.50.img.xz +The easiest way to test out Nethadone is with a board like the +Zero or R1 that has functional wifi with Armbian. You can spin +up a [hotspot](https://ubuntu.com/core/docs/networkmanager/configure-wifi-access-points) with `nmcli`, eg: -This is because we will need to use a BTF-enabled kernel provided -by [daeuniverse](https://github.com/daeuniverse/armbian-btf-kernel), -and that is the latest version of Armbian provided there. +```bash +sudo nmcli dev wifi hotspot ifname wlan0 ssid nethadone password $password channel 8 band bg +``` -### Image prep +The network-manager hotspot conveniently sets up a LAN for you +for any devices that connect wirelessly (default seems to be `10.42.0.0/24`), and NAT out to your wired interface, which should be plugged in +directly to your primary router. -The contents of `scripts/customize-image.sh` will need to be run, -as root, in a fresh installation of the image you downlaoded. -If you want to save time, you can prep this on your machine using -the chroot build script, eg: +> [!NOTE] +> The network-manager hotspot also has the advantage of setting up DNS +> caching automatically with `dnsmasq`. This makes it easier for +> nethadone to capture these packets, as clients receiving a local +> IP as a DNS server in DHCP configuration typically do not use DNS-over-HTTPS. -```bash -cd scripts -./build.sh -i /tmp/Armbian_23.8.1_Orangepi-r1plus_jammy_current_6.1.50.img -m ~/mnt -``` +You can drop the `nmcli` command you used into `/etc/rc.local` to have +it start up on reboot. -Replace the `-i` and `-m` options with wherever you uncompressed the -Armbian image above, and an empty folder to use for a mount point. +### Adjusting cofiguration -`build.sh` is just a chroot wrapper that pre-installs everything -required for nethadone to run; if you prefer to do the installation -of required packages directly, you can copy paste from -`customize-image.sh` after you have flashed the base image using -[Balena etcher](https://etcher.balena.io/) or whichever flashing tool you prefer. +If you want to customize any parameters, or have different interface +names, you can create a `/etc/nethadone.yml` file. +See the sample [config](../config/nethadone.yml) for more details. -### Nethadone installation +## Wired-only Deployment -Once you have the image flashed and the device booted, at this -point you should be ready to install Nethadone. +Faster devices like the OrangePi R1+, as of my last tests, do not have +functional wifi with stock Armbian. This means they need to be introduced +between your wifi router and your device providing internet access using +the two ethernet ports. -First clone the repo as regular user (but one with sudo privilege) +This provides fast, full coverage for all devices in your home, but could take down your internet in the event of an issue with nethadone. +As this is still very much in development, it's recommended to stick +with a secondary access point for now. -```bash -git clone https://github.com/atomic77/nethadone -``` +If you have access to an extra wifi router, this is an example of how +you can still make use of a wired-only device: -The default configuration file should work on an r1plus, and you -can run this to build and launch: +![Secondary network](nethadone-secondary-network.drawio.png) + +In such as case, you will need to set up NAT for the interface +connected to the , eg. if `eth0` is your wan interface: ```bash -make run-root +iptables -t nat -A POSTROUTING -o end0 -j MASQUERADE ``` +*TODO - properly document other steps required for this scenario* + +## Runtime logs If all goes well and nethadone is routing and inspecting traffic, -you should see log entries like: +you should see be able to see log entries like: ```bash +$ journalctl -u nethadone.service --follow ... 2024/07/20 15:48:58 Registering prometheus metrics 2024/07/20 15:48:58 Setting up SimpleLoadAverage policy @@ -161,7 +169,7 @@ There is a basic admin interface where you can configure glob groups, see the currently active policy and inspect currently mapped DNS to IP addresses, it will be running at: -http://r1plus.local:3000/ +http://orangepi-r1.local:3000/ mDNS is configured by default - if this address does not resolve on your network, replace `r1plus.local` with whatever IP address @@ -183,37 +191,15 @@ are being collected there using similar queries used by nethadone: ![prometheus](prometheus.png) -### Routing Configuration & Client setup - -One of the design goals of Nethadone is to introduce minimal or -no client configuration beyond DHCP. The growing use of DNS over -HTTPS (DoH), unfortunately complicates this, as it prevents -Nethadone from using the typical UDP DNS packets to associate -a client-requested domain with an IP address. - -In practice, the best way to work around this is to install -[pi-hole](https://github.com/pi-hole/pi-hole) -on the same device. Most client devices on your network, if provided -a local DNS server in the DHCP configuration, will automatically -switch to UDP-based DNS, and Nethadone will be able to inspect these -packets. Installing pi-hole on a non-raspberry pi is -straight-forward though not officially supported, I have created a -[youtube video with instructions](https://www.youtube.com/watch?v=m-mIglWyFcs) for installation on an orangepi zero that should work fine against a R1plus. - -Alternatively, you can configure browsers to -[disable the use of DoH](https://www.expressvpn.com/support/troubleshooting/disable-dns-over-https/). - - -## Deployment - -The recommended way of starting out with Nethadone is to create -a separate, 'protected' network, potentially using a old -spare wifi router. This is how it looks in my environment: - -![Secondary network](nethadone-secondary-network.drawio.png) +## Other boards -Ideally, the wifi found on a board like the Orange Pi R1+ could -be used directly to create an isolated netwrk, but as of this time, Armbian does not have good support for the hardware. +Armbian does not ship with BTF-enabled kernels out of the box at this +time; the two kernel builds available for Armbian `BOARDFAMILY` +[`rockchip64`](https://github.com/armbian/build/blob/752ba047b3cb600095f981cbabb8be53d12bb9a2/config/boards/orangepi-r1plus.csc#L3) +and [`sunxi`](https://github.com/armbian/build/blob/752ba047b3cb600095f981cbabb8be53d12bb9a2/config/boards/orangepi-r1.csc#L3), are targeted for the Orange Pi R1 Plus +and R1 and Zero respectively. Others with the same family are likely, +but not certain to work. -If you are confident that your setup is good, you can then introduce Nethadone in between your primary access point and -your cable/DSL/fiber provider. +If you are interested in trying out another board and are having +trouble getting a BTF kernel working, please raise an issue and I'll +try adding it to the Github Action. \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh index e9c86a9..b886cfb 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -9,16 +9,33 @@ usage() { echo "Usage: $0 -i -m " echo "" + echo "This script will mount the provided image into the mount point and run" + echo "customize-image.sh on it. If you have downloaded an .xz file from the Armbian" + echo "site it's a good idea to keep the original compressed copy and uncompress the" + echo "image into /dev/shm if you have the memory" + echo "" echo "Options:" - echo " -i Armbian base image file (uncompressed)" - echo " -m Mount point" + echo " -i Armbian base image file to write into (uncompressed)" + echo " -m Mount location to use for chroot " + echo " -a 'aarch64' for orangepi R1+ and similar, 'arm' for opizero/r1, and similar" exit 1 } +cleanup () { +echo "Cleaning up.." + sudo umount ${mnt}/dev + sudo umount --lazy ${mnt} + sudo e2fsck -p -f ${loop}p1 + sudo losetup -d ${loop} +} + +trap cleanup ERR EXIT + img="" mnt="" +arch="" -while getopts ":i:m:" opt; do +while getopts ":i:m:a:" opt; do case $opt in i) img="$OPTARG" @@ -26,6 +43,9 @@ while getopts ":i:m:" opt; do m) mnt="$OPTARG" ;; + a) + arch="$OPTARG" + ;; \?) echo "Invalid option: -$OPTARG" >&2 usage @@ -37,13 +57,14 @@ while getopts ":i:m:" opt; do esac done -if [ -z "$img" ] || [ -z "$mnt" ]; then - echo "Error: Both -i and -m options are required." +if [ -z "$img" ] || [ -z "$mnt" ] || [ -z "$arch" ]; then + echo "Error: All flags are required." usage fi set -e +# After trimming down the image see if we can reduce this to 3 or 2g truncate -s 4G $img loop=$(sudo losetup --partscan --show --nooverlap -f $img) # This may fail on older versions of growpart and require @@ -62,13 +83,15 @@ sudo mount --bind /dev ${mnt}/dev sudo mkdir ${mnt}/tmp/overlay # To enter into chroot w/ arm64 emulation -sudo cp /usr/bin/qemu-aarch64-static ${mnt}/usr/bin/ +sudo cp /usr/bin/qemu-${arch}-static ${mnt}/usr/bin/ sudo cp customize-image.sh ${mnt}/tmp/ + +# Ubuntu Noble images have a symlink to /run/systemd/resolve/stub-resolv.conf which +# breaks DNS when in chroot. Swap out the current users' resolv.conf so we can +# install packages while in chroot +sudo mv ${mnt}/etc/resolv.conf ${mnt}/etc/resolv.conf.tmp +sudo cp -L /etc/resolv.conf ${mnt}/etc/ sudo chmod +x ${mnt}/tmp/customize-image.sh -# https://stackoverflow.com/questions/8157931/bash-executing-commands-from-within-a-chroot-and-switch-user#8157973 -sudo chroot ${mnt} qemu-aarch64-static /bin/bash -c "/tmp/customize-image.sh" +sudo chroot ${mnt} qemu-${arch}-static /bin/bash -c "/tmp/customize-image.sh" -# Clean up -sudo umount ${mnt} -sudo e2fsck -p -f ${loop}p1 -sudo losetup -d ${loop} +sudo mv -f ${mnt}/etc/resolv.conf.tmp ${mnt}/etc/resolv.conf diff --git a/scripts/customize-image.sh b/scripts/customize-image.sh index ae9550d..59a8bd2 100644 --- a/scripts/customize-image.sh +++ b/scripts/customize-image.sh @@ -2,18 +2,17 @@ case "$(arch)" in 'aarch64') - prom_url='https://github.com/prometheus/prometheus/releases/download/v2.53.1/prometheus-2.53.1.linux-arm64.tar.gz' - go_url='https://go.dev/dl/go1.22.3.linux-arm64.tar.gz' - kern_url='https://github.com/daeuniverse/armbian-btf-kernel/releases/download/main-2023-06-17/kernel-rockchip64-current_23.08.0-trunk--6.1.34-Sca87-Dbeb1-Pa401-C3053Hfe66-HK01ba-Vc222-B76dc.tar' + # kern_url='https://github.com/daeuniverse/armbian-btf-kernel/releases/download/main-2023-06-17/kernel-rockchip64-current_23.08.0-trunk--6.1.34-Sca87-Dbeb1-Pa401-C3053Hfe66-HK01ba-Vc222-B76dc.tar' + kern_url='https://github.com/atomic77/nethadone/releases/download/btf-kernel/kernel-legacy-rockchip64-orangepi-r1plus.tar.gz' + neth_url='https://github.com/atomic77/nethadone/releases/download/nethadone-2024-08-12/nethadone-arm64-linux' ;; 'armv7l') - prom_url='https://github.com/prometheus/prometheus/releases/download/v2.53.1/prometheus-2.53.1.linux-armv7.tar.gz' - go_url='https://go.dev/dl/go1.22.3.linux-armv6l.tar.gz' kern_url='https://github.com/atomic77/nethadone/releases/download/btf-kernel/kernel-legacy-sunxi-orangepi-r1.tar.gz' + neth_url='https://github.com/atomic77/nethadone/releases/download/nethadone-2024-08-12/nethadone-arm-linux' ;; 'x86_64') - prom_url='https://github.com/prometheus/prometheus/releases/download/v2.53.1/prometheus-2.53.1.linux-amd64.tar.gz' - go_url='https://go.dev/dl/go1.22.3.linux-arm64.tar.gz' + neth_url='https://github.com/atomic77/nethadone/releases/download/nethadone-2024-08-12/nethadone-amd64-linux' + ;; # Most x86_64 builds for virtual machine testing use should have BTF enabled '*') echo "Unsupported architecture $(arch), exiting." @@ -26,48 +25,15 @@ apt-get update -y apt-get install -y apt-transport-https ca-certificates curl clang llvm jq \ libelf-dev libpcap-dev libbfd-dev binutils-dev build-essential make \ - vim libbpf-dev avahi-daemon linux-tools-common dnsmasq - -wget ${go_url} -rm -rf /usr/local/go && tar -C /usr/local -xzf go1.* -rm go*.tar.gz -echo "export PATH=\$PATH:/usr/local/go/bin" >> /etc/profile - -##### -# Local prometheus for metrics collection -wget ${prom_url} -tar -C /usr/local/bin --strip-components 1 -xzf prometheus-*.tar.gz -rm prometheus-*.tar.gz - -cp /usr/local/bin/prometheus.yml /etc -mkdir -p /var/lib/prometheus + vim libbpf-dev avahi-daemon linux-tools-common dnsmasq prometheus -cat >> /etc/prometheus.yml << EOF +cat >> /etc/prometheus/prometheus.yml << EOF - job_name: "nethadone" static_configs: - targets: ["localhost:3000"] EOF -cat < /etc/systemd/system/prometheus.service -[Unit] -Description=Prometheus -Wants=network-online.target -After=network-online.target -[Service] -User=root -Restart=on-failure - -ExecStart=/usr/local/bin/prometheus \ - --config.file=/etc/prometheus.yml \ - --storage.tsdb.path=/var/lib/prometheus - -[Install] -WantedBy=multi-user.target - -EOF - -## FIXME The interfaces are not being read from config file due to a bug in nethadone.go cat < /etc/systemd/system/nethadone.service [Unit] Description=Nethadone @@ -87,7 +53,7 @@ WantedBy=multi-user.target EOF systemctl daemon-reload -systemctl enable prometheus +systemctl enable nethadone ###### # NAT forwarding will need to be in place for routing to work, @@ -105,7 +71,7 @@ echo "net.ipv4.ip_forward = 1" > /etc/sysctl.d/20-nethadone.conf # ##### -# Grab the custom BTF-enabled kernel from daeuniverse' repo +# Grab the custom BTF-enabled kernel if [ $(arch) != 'x86_64' ]; then wget ${kern_url} tar xvf kernel-* @@ -113,7 +79,7 @@ if [ $(arch) != 'x86_64' ]; then rm kernel*.tar fi -if [ $(arch) == 'armv7' ]; then +if [ $(arch) == 'armv7l' ]; then apt-get install libc6-dev-armel-cross -y fi @@ -126,6 +92,7 @@ make -j 4 make install rm -rf ~/src -# TODO - Copy in pre-built nethadone binary from github to avoid git +# Finally, copy in pre-built nethadone binary from github to avoid git # checkout and golang compiler - +curl -L -o /usr/local/bin/nethadone ${neth_url} +chmod +x /usr/local/bin/nethadone