diff --git a/Jenkinsfile b/Jenkinsfile index aebbbced3af57..ccf21e1eebd44 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -60,15 +60,6 @@ pipeline { } stages { - stage("Load kernel modules") { - steps { - sh ''' - sudo modprobe ip6table_filter - sudo modprobe -va br_netfilter - sudo systemctl restart docker.service - ''' - } - } stage("Print info") { steps { sh 'docker version' diff --git a/hack/make/run b/hack/make/run index ca7eba832f127..16d9febc34275 100644 --- a/hack/make/run +++ b/hack/make/run @@ -91,19 +91,6 @@ if [ -n "$DOCKER_ROOTLESS" ]; then ) fi -# On a host using nftables, the ip6_tables kernel module may need to be loaded. -# This trick is borrowed from the docker (dind) official image ... -# "modprobe" without modprobe -# https://twitter.com/lucabruno/status/902934379835662336 -# This isn't 100% fool-proof, but it'll have a much higher success rate than -# simply using the "real" modprobe (which isn't installed in the dev container). -if ! ip6tables -nL > /dev/null 2>&1; then - ip link show ip6_tables > /dev/null 2>&1 || true - if ! ip6tables -nL > /dev/null 2>&1; then - echo >&2 'ip6tables is not available' - fi -fi - set -x # shellcheck disable=SC2086 exec "${dockerd[@]}" "${args[@]}" diff --git a/internal/modprobe/modprobe_linux.go b/internal/modprobe/modprobe_linux.go new file mode 100644 index 0000000000000..14c677d3d9d83 --- /dev/null +++ b/internal/modprobe/modprobe_linux.go @@ -0,0 +1,110 @@ +// Package modprobe attempts to load kernel modules. It may have more success +// than simply running "modprobe", particularly for docker-in-docker. +package modprobe + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strings" + + "github.com/containerd/log" + "golang.org/x/sys/unix" +) + +// LoadModules attempts to load kernel modules, if necessary. +// +// isLoaded must be a function that checks whether the modules are loaded. It may +// be called multiple times. isLoaded must return an error to indicate that the +// modules still need to be loaded, otherwise nil. +// +// For each method of loading modules, LoadModules will attempt the load for each +// of modNames, then it will call isLoaded to check the result - moving on to try +// the next method if needed, and there is one. +// +// The returned error is the result of the final call to isLoaded. +func LoadModules(ctx context.Context, isLoaded func() error, modNames ...string) error { + if isLoaded() == nil { + log.G(ctx).WithFields(log.Fields{ + "modules": modNames, + }).Debug("Modules already loaded") + return nil + } + + if err := tryLoad(ctx, isLoaded, modNames, ioctlLoader{}); err != nil { + return tryLoad(ctx, isLoaded, modNames, modprobeLoader{}) + } + return nil +} + +type loader interface { + name() string + load(modName string) error +} + +func tryLoad(ctx context.Context, isLoaded func() error, modNames []string, loader loader) error { + var loadErrs []error + for _, modName := range modNames { + if err := loader.load(modName); err != nil { + loadErrs = append(loadErrs, err) + } + } + + if checkResult := isLoaded(); checkResult != nil { + log.G(ctx).WithFields(log.Fields{ + "loader": loader.name(), + "modules": modNames, + "loadErrors": errors.Join(loadErrs...), + "checkResult": checkResult, + }).Debug("Modules not loaded") + return checkResult + } + + log.G(ctx).WithFields(log.Fields{ + "loader": loader.name(), + "modules": modNames, + "loadErrors": errors.Join(loadErrs...), + }).Debug("Modules loaded") + return nil +} + +// ioctlLoader attempts to load the module using an ioctl() to get the interface index +// of a module - it won't have one, but the kernel may load the module. This tends to +// work in docker-in-docker, where the inner-docker may not have "modprobe" or access +// to modules in the host's filesystem. +type ioctlLoader struct{} + +func (il ioctlLoader) name() string { return "ioctl" } + +func (il ioctlLoader) load(modName string) error { + sd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) + if err != nil { + return fmt.Errorf("creating socket for ioctl load of %s: %w", modName, err) + } + defer unix.Close(sd) + + // This tends to work, if running with CAP_SYS_MODULE, because... + // https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L457 + // https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L371-L372 + ifreq, err := unix.NewIfreq(modName) + if err != nil { + return fmt.Errorf("creating ifreq for %s: %w", modName, err) + } + // An error is returned even if the module load is successful. So, ignore it. + _ = unix.IoctlIfreq(sd, unix.SIOCGIFINDEX, ifreq) + return nil +} + +// modprobeLoader attempts to load a kernel module using modprobe. +type modprobeLoader struct{} + +func (ml modprobeLoader) name() string { return "modprobe" } + +func (ml modprobeLoader) load(modName string) error { + out, err := exec.Command("modprobe", "-va", modName).CombinedOutput() + if err != nil { + return fmt.Errorf("modprobe %s failed with message: %q, error: %w", modName, strings.TrimSpace(string(out)), err) + } + return nil +} diff --git a/libnetwork/drivers/bridge/bridge_linux.go b/libnetwork/drivers/bridge/bridge_linux.go index 8b3e83374ad1a..e408cb9b1965d 100644 --- a/libnetwork/drivers/bridge/bridge_linux.go +++ b/libnetwork/drivers/bridge/bridge_linux.go @@ -10,6 +10,7 @@ import ( "github.com/containerd/log" "github.com/docker/docker/errdefs" + "github.com/docker/docker/internal/modprobe" "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/libnetwork/datastore" "github.com/docker/docker/libnetwork/driverapi" @@ -553,6 +554,14 @@ func (d *driver) configure(option map[string]interface{}) error { } if config.EnableIP6Tables { + if err := modprobe.LoadModules(context.TODO(), func() error { + iptable := iptables.GetIptable(iptables.IPv6) + _, err := iptable.Raw("-t", "filter", "-n", "-L", "FORWARD") + return err + }, "ip6_tables"); err != nil { + log.G(context.TODO()).WithError(err).Debug("Loading ip6_tables") + } + removeIPChains(iptables.IPv6) if err := setupHashNetIpset(ipsetExtBridges6, unix.AF_INET6); err != nil { diff --git a/libnetwork/drivers/bridge/setup_bridgenetfiltering.go b/libnetwork/drivers/bridge/setup_bridgenetfiltering.go index 0508a86050ed2..98fd010c993c5 100644 --- a/libnetwork/drivers/bridge/setup_bridgenetfiltering.go +++ b/libnetwork/drivers/bridge/setup_bridgenetfiltering.go @@ -7,10 +7,10 @@ import ( "errors" "fmt" "os" - "os/exec" "syscall" "github.com/containerd/log" + "github.com/docker/docker/internal/modprobe" ) // setupIPv4BridgeNetFiltering checks whether IPv4 forwarding is enabled and, if @@ -46,21 +46,17 @@ func setupIPv6BridgeNetFiltering(config *networkConfiguration, _ *bridgeInterfac } func loadBridgeNetFilterModule(fullPath string) error { - // br_netfilter implictly loads bridge module upon modprobe - modName := "br_netfilter" - if _, err := os.Stat(fullPath); err != nil { - if out, err := exec.Command("modprobe", "-va", modName).CombinedOutput(); err != nil { - log.G(context.TODO()).WithError(err).Errorf("Running modprobe %s failed with message: %s", modName, out) - return fmt.Errorf("cannot restrict inter-container communication: modprobe %s failed: %w", modName, err) - } - } - return nil + // br_netfilter implicitly loads bridge module upon modprobe + return modprobe.LoadModules(context.TODO(), func() error { + _, err := os.Stat(fullPath) + return err + }, "br_netfilter") } // Enable bridge net filtering if not already enabled. See GitHub issue #11404 func enableBridgeNetFiltering(nfParam string) error { if err := loadBridgeNetFilterModule(nfParam); err != nil { - return fmt.Errorf("loadBridgeNetFilterModule failed: %s", err) + return fmt.Errorf("cannot restrict inter-container communication or run without the userland proxy: %w", err) } enabled, err := getKernelBoolParam(nfParam) if err != nil { diff --git a/libnetwork/ns/init_linux.go b/libnetwork/ns/init_linux.go index b2d945e885448..d63c11c56e099 100644 --- a/libnetwork/ns/init_linux.go +++ b/libnetwork/ns/init_linux.go @@ -2,14 +2,12 @@ package ns import ( "context" - "fmt" - "os/exec" - "strings" "sync" "syscall" "time" "github.com/containerd/log" + "github.com/docker/docker/internal/modprobe" "github.com/docker/docker/internal/nlwrap" "github.com/vishvananda/netns" ) @@ -65,12 +63,8 @@ func getSupportedNlFamilies() []int { fams = append(fams, syscall.NETLINK_XFRM) } // NETLINK_NETFILTER test - if err := loadNfConntrackModules(); err != nil { - if checkNfSocket() != nil { - log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err) - } else { - fams = append(fams, syscall.NETLINK_NETFILTER) - } + if err := modprobe.LoadModules(context.TODO(), checkNfSocket, "nf_conntrack", "nf_conntrack_netlink"); err != nil { + log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err) } else { fams = append(fams, syscall.NETLINK_NETFILTER) } @@ -88,16 +82,6 @@ func checkXfrmSocket() error { return nil } -func loadNfConntrackModules() error { - if out, err := exec.Command("modprobe", "-va", "nf_conntrack").CombinedOutput(); err != nil { - return fmt.Errorf("Running modprobe nf_conntrack failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err) - } - if out, err := exec.Command("modprobe", "-va", "nf_conntrack_netlink").CombinedOutput(); err != nil { - return fmt.Errorf("Running modprobe nf_conntrack_netlink failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err) - } - return nil -} - // API check on required nf_conntrack* modules (nf_conntrack, nf_conntrack_netlink) func checkNfSocket() error { fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_NETFILTER)