Skip to content

Commit

Permalink
Merge pull request moby#49038 from robmry/modprobeless
Browse files Browse the repository at this point in the history
Try to load kernel modules, without modprobe
  • Loading branch information
robmry authored Dec 6, 2024
2 parents b1fc766 + 15ba03c commit 4b8c720
Show file tree
Hide file tree
Showing 6 changed files with 129 additions and 52 deletions.
9 changes: 0 additions & 9 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,6 @@ pipeline {
}

stages {
stage("Load kernel modules") {
steps {
sh '''
sudo modprobe ip6table_filter
sudo modprobe -va br_netfilter
sudo systemctl restart docker.service
'''
}
}
stage("Print info") {
steps {
sh 'docker version'
Expand Down
13 changes: 0 additions & 13 deletions hack/make/run
Original file line number Diff line number Diff line change
Expand Up @@ -91,19 +91,6 @@ if [ -n "$DOCKER_ROOTLESS" ]; then
)
fi

# On a host using nftables, the ip6_tables kernel module may need to be loaded.
# This trick is borrowed from the docker (dind) official image ...
# "modprobe" without modprobe
# https://twitter.com/lucabruno/status/902934379835662336
# This isn't 100% fool-proof, but it'll have a much higher success rate than
# simply using the "real" modprobe (which isn't installed in the dev container).
if ! ip6tables -nL > /dev/null 2>&1; then
ip link show ip6_tables > /dev/null 2>&1 || true
if ! ip6tables -nL > /dev/null 2>&1; then
echo >&2 'ip6tables is not available'
fi
fi

set -x
# shellcheck disable=SC2086
exec "${dockerd[@]}" "${args[@]}"
110 changes: 110 additions & 0 deletions internal/modprobe/modprobe_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Package modprobe attempts to load kernel modules. It may have more success
// than simply running "modprobe", particularly for docker-in-docker.
package modprobe

import (
"context"
"errors"
"fmt"
"os/exec"
"strings"

"github.com/containerd/log"
"golang.org/x/sys/unix"
)

// LoadModules attempts to load kernel modules, if necessary.
//
// isLoaded must be a function that checks whether the modules are loaded. It may
// be called multiple times. isLoaded must return an error to indicate that the
// modules still need to be loaded, otherwise nil.
//
// For each method of loading modules, LoadModules will attempt the load for each
// of modNames, then it will call isLoaded to check the result - moving on to try
// the next method if needed, and there is one.
//
// The returned error is the result of the final call to isLoaded.
func LoadModules(ctx context.Context, isLoaded func() error, modNames ...string) error {
if isLoaded() == nil {
log.G(ctx).WithFields(log.Fields{
"modules": modNames,
}).Debug("Modules already loaded")
return nil
}

if err := tryLoad(ctx, isLoaded, modNames, ioctlLoader{}); err != nil {
return tryLoad(ctx, isLoaded, modNames, modprobeLoader{})
}
return nil
}

type loader interface {
name() string
load(modName string) error
}

func tryLoad(ctx context.Context, isLoaded func() error, modNames []string, loader loader) error {
var loadErrs []error
for _, modName := range modNames {
if err := loader.load(modName); err != nil {
loadErrs = append(loadErrs, err)
}
}

if checkResult := isLoaded(); checkResult != nil {
log.G(ctx).WithFields(log.Fields{
"loader": loader.name(),
"modules": modNames,
"loadErrors": errors.Join(loadErrs...),
"checkResult": checkResult,
}).Debug("Modules not loaded")
return checkResult
}

log.G(ctx).WithFields(log.Fields{
"loader": loader.name(),
"modules": modNames,
"loadErrors": errors.Join(loadErrs...),
}).Debug("Modules loaded")
return nil
}

// ioctlLoader attempts to load the module using an ioctl() to get the interface index
// of a module - it won't have one, but the kernel may load the module. This tends to
// work in docker-in-docker, where the inner-docker may not have "modprobe" or access
// to modules in the host's filesystem.
type ioctlLoader struct{}

func (il ioctlLoader) name() string { return "ioctl" }

func (il ioctlLoader) load(modName string) error {
sd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
if err != nil {
return fmt.Errorf("creating socket for ioctl load of %s: %w", modName, err)
}
defer unix.Close(sd)

// This tends to work, if running with CAP_SYS_MODULE, because...
// https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L457
// https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L371-L372
ifreq, err := unix.NewIfreq(modName)
if err != nil {
return fmt.Errorf("creating ifreq for %s: %w", modName, err)
}
// An error is returned even if the module load is successful. So, ignore it.
_ = unix.IoctlIfreq(sd, unix.SIOCGIFINDEX, ifreq)
return nil
}

// modprobeLoader attempts to load a kernel module using modprobe.
type modprobeLoader struct{}

func (ml modprobeLoader) name() string { return "modprobe" }

func (ml modprobeLoader) load(modName string) error {
out, err := exec.Command("modprobe", "-va", modName).CombinedOutput()
if err != nil {
return fmt.Errorf("modprobe %s failed with message: %q, error: %w", modName, strings.TrimSpace(string(out)), err)
}
return nil
}
9 changes: 9 additions & 0 deletions libnetwork/drivers/bridge/bridge_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/containerd/log"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/internal/modprobe"
"github.com/docker/docker/internal/nlwrap"
"github.com/docker/docker/libnetwork/datastore"
"github.com/docker/docker/libnetwork/driverapi"
Expand Down Expand Up @@ -553,6 +554,14 @@ func (d *driver) configure(option map[string]interface{}) error {
}

if config.EnableIP6Tables {
if err := modprobe.LoadModules(context.TODO(), func() error {
iptable := iptables.GetIptable(iptables.IPv6)
_, err := iptable.Raw("-t", "filter", "-n", "-L", "FORWARD")
return err
}, "ip6_tables"); err != nil {
log.G(context.TODO()).WithError(err).Debug("Loading ip6_tables")
}

removeIPChains(iptables.IPv6)

if err := setupHashNetIpset(ipsetExtBridges6, unix.AF_INET6); err != nil {
Expand Down
18 changes: 7 additions & 11 deletions libnetwork/drivers/bridge/setup_bridgenetfiltering.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import (
"errors"
"fmt"
"os"
"os/exec"
"syscall"

"github.com/containerd/log"
"github.com/docker/docker/internal/modprobe"
)

// setupIPv4BridgeNetFiltering checks whether IPv4 forwarding is enabled and, if
Expand Down Expand Up @@ -46,21 +46,17 @@ func setupIPv6BridgeNetFiltering(config *networkConfiguration, _ *bridgeInterfac
}

func loadBridgeNetFilterModule(fullPath string) error {
// br_netfilter implictly loads bridge module upon modprobe
modName := "br_netfilter"
if _, err := os.Stat(fullPath); err != nil {
if out, err := exec.Command("modprobe", "-va", modName).CombinedOutput(); err != nil {
log.G(context.TODO()).WithError(err).Errorf("Running modprobe %s failed with message: %s", modName, out)
return fmt.Errorf("cannot restrict inter-container communication: modprobe %s failed: %w", modName, err)
}
}
return nil
// br_netfilter implicitly loads bridge module upon modprobe
return modprobe.LoadModules(context.TODO(), func() error {
_, err := os.Stat(fullPath)
return err
}, "br_netfilter")
}

// Enable bridge net filtering if not already enabled. See GitHub issue #11404
func enableBridgeNetFiltering(nfParam string) error {
if err := loadBridgeNetFilterModule(nfParam); err != nil {
return fmt.Errorf("loadBridgeNetFilterModule failed: %s", err)
return fmt.Errorf("cannot restrict inter-container communication or run without the userland proxy: %w", err)
}
enabled, err := getKernelBoolParam(nfParam)
if err != nil {
Expand Down
22 changes: 3 additions & 19 deletions libnetwork/ns/init_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,12 @@ package ns

import (
"context"
"fmt"
"os/exec"
"strings"
"sync"
"syscall"
"time"

"github.com/containerd/log"
"github.com/docker/docker/internal/modprobe"
"github.com/docker/docker/internal/nlwrap"
"github.com/vishvananda/netns"
)
Expand Down Expand Up @@ -65,12 +63,8 @@ func getSupportedNlFamilies() []int {
fams = append(fams, syscall.NETLINK_XFRM)
}
// NETLINK_NETFILTER test
if err := loadNfConntrackModules(); err != nil {
if checkNfSocket() != nil {
log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err)
} else {
fams = append(fams, syscall.NETLINK_NETFILTER)
}
if err := modprobe.LoadModules(context.TODO(), checkNfSocket, "nf_conntrack", "nf_conntrack_netlink"); err != nil {
log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err)
} else {
fams = append(fams, syscall.NETLINK_NETFILTER)
}
Expand All @@ -88,16 +82,6 @@ func checkXfrmSocket() error {
return nil
}

func loadNfConntrackModules() error {
if out, err := exec.Command("modprobe", "-va", "nf_conntrack").CombinedOutput(); err != nil {
return fmt.Errorf("Running modprobe nf_conntrack failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)
}
if out, err := exec.Command("modprobe", "-va", "nf_conntrack_netlink").CombinedOutput(); err != nil {
return fmt.Errorf("Running modprobe nf_conntrack_netlink failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)
}
return nil
}

// API check on required nf_conntrack* modules (nf_conntrack, nf_conntrack_netlink)
func checkNfSocket() error {
fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_NETFILTER)
Expand Down

0 comments on commit 4b8c720

Please sign in to comment.