Skip to content

Commit

Permalink
add iptables-wrapper-install.sh to release-1.9
Browse files Browse the repository at this point in the history
  • Loading branch information
changluyi committed Oct 23, 2023
1 parent 574f9bb commit 438855a
Show file tree
Hide file tree
Showing 10 changed files with 405 additions and 46 deletions.
1 change: 1 addition & 0 deletions dist/images/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ COPY logrotate/* /etc/logrotate.d/
COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller

WORKDIR /kube-ovn
RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check

RUN rm -f /usr/bin/nc &&\
rm -f /usr/bin/netcat
Expand Down
3 changes: 0 additions & 3 deletions dist/images/Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,6 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname
tcpdump ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 \
logrotate dnsutils net-tools nmap -y --no-install-recommends && \
rm -rf /var/lib/apt/lists/* && \
cd /usr/sbin && \
ln -sf /usr/sbin/iptables-legacy iptables && \
ln -sf /usr/sbin/ip6tables-legacy ip6tables && \
rm -rf /etc/localtime

RUN mkdir -p /var/run/openvswitch && \
Expand Down
211 changes: 211 additions & 0 deletions dist/images/iptables-wrapper-installer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
#!/bin/sh

# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Usage:
#
# iptables-wrapper-installer.sh [--no-sanity-check]
#
# Installs a wrapper iptables script in a container that will figure out
# whether iptables-legacy or iptables-nft is in use on the host and then
# replaces itself with the correct underlying iptables version.
#
# Unless "--no-sanity-check" is passed, it will first verify that the
# container already contains a suitable version of iptables.

# NOTE: This can only use POSIX /bin/sh features; the build container
# might not contain bash.

# original source:
# https://github.com/kubernetes-sigs/iptables-wrappers/blob/master/iptables-wrapper-installer.sh

set -eu

# Find iptables binary location
if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then
sbin="/usr/sbin"
elif [ -d /sbin -a -e /sbin/iptables ]; then
sbin="/sbin"
else
echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2
exit 1
fi

# Determine how the system selects between iptables-legacy and iptables-nft
if [ -x /usr/sbin/alternatives ]; then
# Fedora/SUSE style alternatives
altstyle="fedora"
elif [ -x /usr/sbin/update-alternatives ]; then
# Debian style alternatives
altstyle="debian"
else
# No alternatives system
altstyle="none"
fi

if [ "${1:-}" != "--no-sanity-check" ]; then
# Ensure dependencies are installed
if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then
echo "ERROR: iptables-nft is not installed" 1>&2
exit 1
fi
if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then
echo "ERROR: iptables-legacy is not installed" 1>&2
exit 1
fi

case "${version}" in
*v1.8.[0123]\ *)
echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2
echo " Upgrade to 1.8.4 or newer." 1>&2
exit 1
;;
*)
# 1.8.4+ are OK
;;
esac
fi

# Start creating the wrapper...
rm -f "${sbin}/iptables-wrapper"
cat > "${sbin}/iptables-wrapper" <<EOF
#!/bin/sh
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE: This can only use POSIX /bin/sh features; the container image
# might not contain bash.
set -eu
# In kubernetes 1.17 and later, kubelet will have created at least
# one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
# "KUBE-KUBELET-CANARY"), so check that first, against
# iptables-nft, because we can check that more efficiently and
# it's more common these days.
nft_kubelet_rules=\$( (iptables-nft-save -t mangle || true; ip6tables-nft-save -t mangle || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${nft_kubelet_rules}" -ne 0 ]; then
mode=nft
else
# Check for kubernetes 1.17-or-later with iptables-legacy. We
# can't pass "-t mangle" to iptables-legacy-save because it would
# cause the kernel to create that table if it didn't already
# exist, which we don't want. So we have to grab all the rules
legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${legacy_kubelet_rules}" -ne 0 ]; then
mode=legacy
else
# With older kubernetes releases there may not be any _specific_
# rules we can look for, but we assume that some non-containerized process
# (possibly kubelet) will have created _some_ iptables rules.
num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
mode=legacy
else
mode=nft
fi
fi
fi
EOF

# Write out the appropriate alternatives-selection commands
case "${altstyle}" in
fedora)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
EOF
;;

debian)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
update-alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1
EOF
;;

*)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/\${cmd}"
ln -s "${sbin}/xtables-\${mode}-multi" "${sbin}/\${cmd}"
done 2>/dev/null || failed=1
EOF
;;
esac

# Write out the post-alternatives-selection error checking and final wrap-up
cat >> "${sbin}/iptables-wrapper" <<EOF
if [ "\${failed:-0}" = 1 ]; then
echo "Unable to redirect iptables binaries. (Are you running in an unprivileged pod?)" 1>&2
# fake it, though this will probably also fail if they aren't root
exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@"
fi
# Now re-exec the original command with the newly-selected alternative
exec "\$0" "\$@"
EOF
chmod +x "${sbin}/iptables-wrapper"

# Now back in the installer script, point the iptables binaries at our
# wrapper
case "${altstyle}" in
fedora)
alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper
;;

debian)
update-alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper
update-alternatives \
--install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper
;;

*)
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/${cmd}"
ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}"
done
;;
esac

# Cleanup
rm -f "$0"
3 changes: 3 additions & 0 deletions dist/images/start-cniserver.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ while true; do
sleep 1
done

# update links to point to the iptables binaries
iptables -V

# If nftables not exist do not exit
set +e
iptables -P FORWARD ACCEPT
Expand Down
3 changes: 3 additions & 0 deletions dist/images/start-ovs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ function quit {
}
trap quit EXIT

# update links to point to the iptables binaries
iptables -V

# Start ovsdb
/usr/share/openvswitch/scripts/ovs-ctl restart --no-ovs-vswitchd --system-id=random
# Restrict the number of pthreads ovs-vswitchd creates to reduce the
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ require (
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08
github.com/containernetworking/cni v0.8.1
github.com/containernetworking/plugins v0.9.1
github.com/coreos/go-iptables v0.6.0
github.com/emicklei/go-restful/v3 v3.9.0
github.com/evanphx/json-patch v5.6.0+incompatible
github.com/golang/protobuf v1.5.2
github.com/greenpau/ovsdb v0.0.0-20181114004433-3582b85e8968
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589
github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
github.com/neverlee/keymutex v0.0.0-20171121013845-f593aa834bf9
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,6 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
Expand Down Expand Up @@ -693,6 +691,8 @@ github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c h1:AcOKlV+lInNlGO3o3+1
github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c/go.mod h1:Ce8lvkopTGXfPmeb5AY3/umEOmoFVV3HlCPGfGk0+Y0=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139 h1:MaLC8/dohKHU8nkfglfE2oikefB6urJG75yZDOcKTRU=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139/go.mod h1:ulxnUH9cbIOtCH+exhJPeV2mleh+bDv67WKsl/MVU/g=
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca h1:fTMjoho2et9nKVOFrjzVEWVd9XD1zzxOYrlxxZpO0fU=
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca/go.mod h1:jY1XeGzkx8ASNJ+SqQSxTESNXARkjvt+I6IJOTnzIjw=
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589 h1:y9exo1hjCsq7jsGUzt11kxhTiEGrGSQ0ZqibAiZk2PQ=
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589/go.mod h1:49upX+/hUyppWIqu58cumojyIwXdkA8k6reA/mQlKuI=
github.com/kubeovn/libovsdb v0.0.0-20221125061852-8b910935f8e4 h1:S/R2b2/S7L3l68Y2YwV/0zDGEVDargfRVqYu6989fIw=
Expand Down
65 changes: 57 additions & 8 deletions pkg/daemon/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net"
"os"
"os/exec"
"path/filepath"
"reflect"
"strconv"
"strings"
Expand All @@ -16,7 +17,7 @@ import (
"time"

"github.com/alauda/felix/ipsets"
"github.com/coreos/go-iptables/iptables"
"github.com/kubeovn/go-iptables/iptables"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -66,9 +67,10 @@ type Controller struct {

recorder record.EventRecorder

iptables map[string]*iptables.IPTables
ipsets map[string]*ipsets.IPSets
ipsetLock sync.Mutex
iptables map[string]*iptables.IPTables
iptablesObsolete map[string]*iptables.IPTables
ipsets map[string]*ipsets.IPSets
ipsetLock sync.Mutex

protocol string
localPodName string
Expand Down Expand Up @@ -121,22 +123,44 @@ func NewController(config *Configuration, podInformerFactory informers.SharedInf
}
controller.protocol = util.CheckProtocol(node.Annotations[util.IpAddressAnnotation])

ok, err := isLegacyIptablesMode()
if err != nil {
klog.Errorf("failed to check iptables mode: %v", err)
return nil, err
}
if !ok {
// iptables works in nft mode, we should migrate iptables rules
controller.iptablesObsolete = make(map[string]*iptables.IPTables, 2)
}

controller.iptables = make(map[string]*iptables.IPTables)
controller.ipsets = make(map[string]*ipsets.IPSets)
if controller.protocol == kubeovnv1.ProtocolIPv4 || controller.protocol == kubeovnv1.ProtocolDual {
iptables, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
if err != nil {
return nil, err
}
controller.iptables[kubeovnv1.ProtocolIPv4] = iptables
controller.iptables[kubeovnv1.ProtocolIPv4] = ipt
if controller.iptablesObsolete != nil {
if ipt, err = iptables.NewWithProtocolAndMode(iptables.ProtocolIPv4, "legacy"); err != nil {
return nil, err
}
controller.iptablesObsolete[kubeovnv1.ProtocolIPv4] = ipt
}
controller.ipsets[kubeovnv1.ProtocolIPv4] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV4, IPSetPrefix, nil, nil))
}
if controller.protocol == kubeovnv1.ProtocolIPv6 || controller.protocol == kubeovnv1.ProtocolDual {
iptables, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
if err != nil {
return nil, err
}
controller.iptables[kubeovnv1.ProtocolIPv6] = iptables
controller.iptables[kubeovnv1.ProtocolIPv6] = ipt
if controller.iptablesObsolete != nil {
if ipt, err = iptables.NewWithProtocolAndMode(iptables.ProtocolIPv6, "legacy"); err != nil {
return nil, err
}
controller.iptablesObsolete[kubeovnv1.ProtocolIPv6] = ipt
}
controller.ipsets[kubeovnv1.ProtocolIPv6] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV6, IPSetPrefix, nil, nil))
}

Expand Down Expand Up @@ -489,6 +513,31 @@ func (c *Controller) processNextSubnetWorkItem() bool {
return true
}

func evalCommandSymlinks(cmd string) (string, error) {
path, err := exec.LookPath(cmd)
if err != nil {
return "", fmt.Errorf("failed to search for command %q: %v", cmd, err)
}
file, err := filepath.EvalSymlinks(path)
if err != nil {
return "", fmt.Errorf("failed to read evaluate symbolic links for file %q: %v", path, err)
}

return file, nil
}

func isLegacyIptablesMode() (bool, error) {
path, err := evalCommandSymlinks("iptables")
if err != nil {
return false, err
}
pathLegacy, err := evalCommandSymlinks("iptables-legacy")
if err != nil {
return false, err
}
return path == pathLegacy, nil
}

func (c *Controller) reconcileRouters(event subnetEvent) error {
subnets, err := c.subnetsLister.List(labels.Everything())
if err != nil {
Expand Down
Loading

0 comments on commit 438855a

Please sign in to comment.