diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index 6f826e0c236..1b5e9851e8e 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -15,6 +15,7 @@ COPY logrotate/* /etc/logrotate.d/ COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller WORKDIR /kube-ovn +RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check RUN rm -f /usr/bin/nc &&\ rm -f /usr/bin/netcat diff --git a/dist/images/Dockerfile.base b/dist/images/Dockerfile.base index 7d6da57c2cd..743382d4aa1 100644 --- a/dist/images/Dockerfile.base +++ b/dist/images/Dockerfile.base @@ -85,9 +85,6 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname tcpdump ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 \ logrotate dnsutils net-tools nmap -y --no-install-recommends && \ rm -rf /var/lib/apt/lists/* && \ - cd /usr/sbin && \ - ln -sf /usr/sbin/iptables-legacy iptables && \ - ln -sf /usr/sbin/ip6tables-legacy ip6tables && \ rm -rf /etc/localtime RUN mkdir -p /var/run/openvswitch && \ diff --git a/dist/images/iptables-wrapper-installer.sh b/dist/images/iptables-wrapper-installer.sh new file mode 100755 index 00000000000..8a650434740 --- /dev/null +++ b/dist/images/iptables-wrapper-installer.sh @@ -0,0 +1,211 @@ +#!/bin/sh + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# +# iptables-wrapper-installer.sh [--no-sanity-check] +# +# Installs a wrapper iptables script in a container that will figure out +# whether iptables-legacy or iptables-nft is in use on the host and then +# replaces itself with the correct underlying iptables version. +# +# Unless "--no-sanity-check" is passed, it will first verify that the +# container already contains a suitable version of iptables. + +# NOTE: This can only use POSIX /bin/sh features; the build container +# might not contain bash. + +# original source: +# https://github.com/kubernetes-sigs/iptables-wrappers/blob/master/iptables-wrapper-installer.sh + +set -eu + +# Find iptables binary location +if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then + sbin="/usr/sbin" +elif [ -d /sbin -a -e /sbin/iptables ]; then + sbin="/sbin" +else + echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2 + exit 1 +fi + +# Determine how the system selects between iptables-legacy and iptables-nft +if [ -x /usr/sbin/alternatives ]; then + # Fedora/SUSE style alternatives + altstyle="fedora" +elif [ -x /usr/sbin/update-alternatives ]; then + # Debian style alternatives + altstyle="debian" +else + # No alternatives system + altstyle="none" +fi + +if [ "${1:-}" != "--no-sanity-check" ]; then + # Ensure dependencies are installed + if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then + echo "ERROR: iptables-nft is not installed" 1>&2 + exit 1 + fi + if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then + echo "ERROR: iptables-legacy is not installed" 1>&2 + exit 1 + fi + + case "${version}" in + *v1.8.[0123]\ *) + echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2 + echo " Upgrade to 1.8.4 or newer." 1>&2 + exit 1 + ;; + *) + # 1.8.4+ are OK + ;; + esac +fi + +# Start creating the wrapper... +rm -f "${sbin}/iptables-wrapper" +cat > "${sbin}/iptables-wrapper" </dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l) +if [ "\${nft_kubelet_rules}" -ne 0 ]; then + mode=nft +else + # Check for kubernetes 1.17-or-later with iptables-legacy. We + # can't pass "-t mangle" to iptables-legacy-save because it would + # cause the kernel to create that table if it didn't already + # exist, which we don't want. So we have to grab all the rules + legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l) + if [ "\${legacy_kubelet_rules}" -ne 0 ]; then + mode=legacy + else + # With older kubernetes releases there may not be any _specific_ + # rules we can look for, but we assume that some non-containerized process + # (possibly kubelet) will have created _some_ iptables rules. + num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l) + num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l) + if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then + mode=legacy + else + mode=nft + fi + fi +fi + +EOF + +# Write out the appropriate alternatives-selection commands +case "${altstyle}" in + fedora) +cat >> "${sbin}/iptables-wrapper" < /dev/null || failed=1 +EOF + ;; + + debian) +cat >> "${sbin}/iptables-wrapper" < /dev/null || failed=1 +update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1 +EOF + ;; + + *) +cat >> "${sbin}/iptables-wrapper" </dev/null || failed=1 +EOF + ;; +esac + +# Write out the post-alternatives-selection error checking and final wrap-up +cat >> "${sbin}/iptables-wrapper" <&2 + # fake it, though this will probably also fail if they aren't root + exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@" +fi + +# Now re-exec the original command with the newly-selected alternative +exec "\$0" "\$@" +EOF +chmod +x "${sbin}/iptables-wrapper" + +# Now back in the installer script, point the iptables binaries at our +# wrapper +case "${altstyle}" in + fedora) + alternatives \ + --install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \ + --slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper + ;; + + debian) + update-alternatives \ + --install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \ + --slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper + update-alternatives \ + --install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \ + --slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper + ;; + + *) + for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do + rm -f "${sbin}/${cmd}" + ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}" + done + ;; +esac + +# Cleanup +rm -f "$0" diff --git a/dist/images/start-cniserver.sh b/dist/images/start-cniserver.sh index 64effd22461..810426571bf 100755 --- a/dist/images/start-cniserver.sh +++ b/dist/images/start-cniserver.sh @@ -38,6 +38,9 @@ while true; do sleep 1 done +# update links to point to the iptables binaries +iptables -V + # If nftables not exist do not exit set +e iptables -P FORWARD ACCEPT diff --git a/dist/images/start-ovs.sh b/dist/images/start-ovs.sh index 3eac5d8f6c7..7a1ad2927d7 100755 --- a/dist/images/start-ovs.sh +++ b/dist/images/start-ovs.sh @@ -51,6 +51,9 @@ function quit { } trap quit EXIT +# update links to point to the iptables binaries +iptables -V + # Start ovsdb /usr/share/openvswitch/scripts/ovs-ctl restart --no-ovs-vswitchd --system-id=random # Restrict the number of pthreads ovs-vswitchd creates to reduce the diff --git a/go.mod b/go.mod index 4b953b8cca1..2c05b4b777e 100644 --- a/go.mod +++ b/go.mod @@ -9,12 +9,12 @@ require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 - github.com/coreos/go-iptables v0.6.0 github.com/emicklei/go-restful/v3 v3.9.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/golang/protobuf v1.5.2 github.com/greenpau/ovsdb v0.0.0-20181114004433-3582b85e8968 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05 + github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589 github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 github.com/neverlee/keymutex v0.0.0-20171121013845-f593aa834bf9 diff --git a/go.sum b/go.sum index 20467f42e13..29f23ad2e8c 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,6 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -693,6 +691,8 @@ github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c h1:AcOKlV+lInNlGO3o3+1 github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c/go.mod h1:Ce8lvkopTGXfPmeb5AY3/umEOmoFVV3HlCPGfGk0+Y0= github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139 h1:MaLC8/dohKHU8nkfglfE2oikefB6urJG75yZDOcKTRU= github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139/go.mod h1:ulxnUH9cbIOtCH+exhJPeV2mleh+bDv67WKsl/MVU/g= +github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca h1:fTMjoho2et9nKVOFrjzVEWVd9XD1zzxOYrlxxZpO0fU= +github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca/go.mod h1:jY1XeGzkx8ASNJ+SqQSxTESNXARkjvt+I6IJOTnzIjw= github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589 h1:y9exo1hjCsq7jsGUzt11kxhTiEGrGSQ0ZqibAiZk2PQ= github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589/go.mod h1:49upX+/hUyppWIqu58cumojyIwXdkA8k6reA/mQlKuI= github.com/kubeovn/libovsdb v0.0.0-20221125061852-8b910935f8e4 h1:S/R2b2/S7L3l68Y2YwV/0zDGEVDargfRVqYu6989fIw= diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go index daed4aeb87d..5c7ace55561 100644 --- a/pkg/daemon/controller.go +++ b/pkg/daemon/controller.go @@ -8,6 +8,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "reflect" "strconv" "strings" @@ -16,7 +17,7 @@ import ( "time" "github.com/alauda/felix/ipsets" - "github.com/coreos/go-iptables/iptables" + "github.com/kubeovn/go-iptables/iptables" "github.com/vishvananda/netlink" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -66,9 +67,10 @@ type Controller struct { recorder record.EventRecorder - iptables map[string]*iptables.IPTables - ipsets map[string]*ipsets.IPSets - ipsetLock sync.Mutex + iptables map[string]*iptables.IPTables + iptablesObsolete map[string]*iptables.IPTables + ipsets map[string]*ipsets.IPSets + ipsetLock sync.Mutex protocol string localPodName string @@ -121,22 +123,44 @@ func NewController(config *Configuration, podInformerFactory informers.SharedInf } controller.protocol = util.CheckProtocol(node.Annotations[util.IpAddressAnnotation]) + ok, err := isLegacyIptablesMode() + if err != nil { + klog.Errorf("failed to check iptables mode: %v", err) + return nil, err + } + if !ok { + // iptables works in nft mode, we should migrate iptables rules + controller.iptablesObsolete = make(map[string]*iptables.IPTables, 2) + } + controller.iptables = make(map[string]*iptables.IPTables) controller.ipsets = make(map[string]*ipsets.IPSets) if controller.protocol == kubeovnv1.ProtocolIPv4 || controller.protocol == kubeovnv1.ProtocolDual { - iptables, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) if err != nil { return nil, err } - controller.iptables[kubeovnv1.ProtocolIPv4] = iptables + controller.iptables[kubeovnv1.ProtocolIPv4] = ipt + if controller.iptablesObsolete != nil { + if ipt, err = iptables.NewWithProtocolAndMode(iptables.ProtocolIPv4, "legacy"); err != nil { + return nil, err + } + controller.iptablesObsolete[kubeovnv1.ProtocolIPv4] = ipt + } controller.ipsets[kubeovnv1.ProtocolIPv4] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV4, IPSetPrefix, nil, nil)) } if controller.protocol == kubeovnv1.ProtocolIPv6 || controller.protocol == kubeovnv1.ProtocolDual { - iptables, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) + ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) if err != nil { return nil, err } - controller.iptables[kubeovnv1.ProtocolIPv6] = iptables + controller.iptables[kubeovnv1.ProtocolIPv6] = ipt + if controller.iptablesObsolete != nil { + if ipt, err = iptables.NewWithProtocolAndMode(iptables.ProtocolIPv6, "legacy"); err != nil { + return nil, err + } + controller.iptablesObsolete[kubeovnv1.ProtocolIPv6] = ipt + } controller.ipsets[kubeovnv1.ProtocolIPv6] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV6, IPSetPrefix, nil, nil)) } @@ -489,6 +513,31 @@ func (c *Controller) processNextSubnetWorkItem() bool { return true } +func evalCommandSymlinks(cmd string) (string, error) { + path, err := exec.LookPath(cmd) + if err != nil { + return "", fmt.Errorf("failed to search for command %q: %v", cmd, err) + } + file, err := filepath.EvalSymlinks(path) + if err != nil { + return "", fmt.Errorf("failed to read evaluate symbolic links for file %q: %v", path, err) + } + + return file, nil +} + +func isLegacyIptablesMode() (bool, error) { + path, err := evalCommandSymlinks("iptables") + if err != nil { + return false, err + } + pathLegacy, err := evalCommandSymlinks("iptables-legacy") + if err != nil { + return false, err + } + return path == pathLegacy, nil +} + func (c *Controller) reconcileRouters(event subnetEvent) error { subnets, err := c.subnetsLister.List(labels.Everything()) if err != nil { diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go index 00d7c42f805..45c9f6e5708 100644 --- a/pkg/daemon/gateway.go +++ b/pkg/daemon/gateway.go @@ -11,6 +11,7 @@ import ( "syscall" "github.com/alauda/felix/ipsets" + "github.com/kubeovn/go-iptables/iptables" "github.com/vishvananda/netlink" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,6 +33,14 @@ const ( IPSetPrefix = "ovn" ) +const ( + NAT = "nat" + Prerouting = "PREROUTING" + Postrouting = "POSTROUTING" + OvnPrerouting = "OVN-PREROUTING" + OvnPostrouting = "OVN-POSTROUTING" +) + type policyRouteMeta struct { family int source string @@ -372,15 +381,6 @@ func (c *Controller) setIptables() error { klog.V(3).Infof("centralized subnets nat ips %v", centralGwNatips) var ( - v4AbandonedRules = []util.IPTableRule{ - {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)}, - {Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x40000/0x40000`)}, - } - v6AbandonedRules = []util.IPTableRule{ - {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)}, - {Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x40000/0x40000`)}, - } - v4Rules = []util.IPTableRule{ // nat service traffic {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE`)}, @@ -435,9 +435,11 @@ func (c *Controller) setIptables() error { } for _, protocol := range protocols { - if c.iptables[protocol] == nil { + ipt := c.iptables[protocol] + if ipt == nil { continue } + // delete unused iptables rule when nat gw with designative ip has been changed in centralized subnet if err = c.deleteUnusedIptablesRule(protocol, "nat", "POSTROUTING", centralGwNatips); err != nil { klog.Errorf("failed to delete iptables rule on node %s, maybe can delete manually, %v", c.config.NodeName, err) @@ -445,19 +447,19 @@ func (c *Controller) setIptables() error { } var matchset string - var abandonedRules, iptablesRules []util.IPTableRule + var obsoleteRules, iptablesRules []util.IPTableRule if protocol == kubeovnv1.ProtocolIPv4 { - iptablesRules, abandonedRules = v4Rules, v4AbandonedRules + iptablesRules = v4Rules matchset = "ovn40subnets" } else { - iptablesRules, abandonedRules = v6Rules, v6AbandonedRules + iptablesRules = v6Rules matchset = "ovn60subnets" } if nodeIP := nodeIPs[protocol]; nodeIP != "" { - abandonedRules = append(abandonedRules, - util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset))}, - ) + obsoleteRules = []util.IPTableRule{ + {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset))}, + } rules := make([]util.IPTableRule, len(iptablesRules)+2) copy(rules[1:4], iptablesRules[:3]) @@ -467,22 +469,6 @@ func (c *Controller) setIptables() error { iptablesRules = rules } - // delete abandoned iptables rules - for _, iptRule := range abandonedRules { - exists, err := c.iptables[protocol].Exists(iptRule.Table, iptRule.Chain, iptRule.Rule...) - if err != nil { - klog.Errorf("failed to check existence of iptables rule: %v", err) - return err - } - if exists { - klog.Infof("deleting abandoned iptables rule: %s", strings.Join(iptRule.Rule, " ")) - if err := c.iptables[protocol].Delete(iptRule.Table, iptRule.Chain, iptRule.Rule...); err != nil { - klog.Errorf("failed to delete iptables rule %s: %v", strings.Join(iptRule.Rule, " "), err) - return err - } - } - } - // add iptables rule for nat gw with designative ip in centralized subnet for cidr, natip := range centralGwNatips { if util.CheckProtocol(cidr) != protocol { @@ -524,6 +510,91 @@ func (c *Controller) setIptables() error { } klog.V(3).Infof("iptables rules %v, exists %v", strings.Join(iptRule.Rule, " "), exists) } + + if err = c.cleanObsoleteIptablesRules(protocol, obsoleteRules); err != nil { + klog.Errorf("failed to clean legacy iptables rules: %v", err) + return err + } + } + + return nil +} + +func deleteIptablesRule(ipt *iptables.IPTables, rule util.IPTableRule) error { + if err := ipt.DeleteIfExists(rule.Table, rule.Chain, rule.Rule...); err != nil { + klog.Errorf("failed to delete iptables rule %q: %v", strings.Join(rule.Rule, " "), err) + return err + } + return nil +} + +func clearObsoleteIptablesChain(ipt *iptables.IPTables, table, chain, parent string) error { + exists, err := ipt.ChainExists(table, chain) + if err != nil { + klog.Error(err) + return err + } + if !exists { + return nil + } + + rule := fmt.Sprintf(`-m comment --comment "kube-ovn %s rules" -j %s`, strings.ToLower(parent), chain) + if err = deleteIptablesRule(ipt, util.IPTableRule{Table: table, Chain: parent, Rule: util.DoubleQuotedFields(rule)}); err != nil { + klog.Error(err) + return err + } + if err = ipt.ClearAndDeleteChain(table, chain); err != nil { + klog.Errorf("failed to delete iptables chain %q in table %s: %v", chain, table, err) + return err + } + return nil +} + +func (c *Controller) cleanObsoleteIptablesRules(protocol string, rules []util.IPTableRule) error { + if c.iptablesObsolete == nil || c.iptablesObsolete[protocol] == nil { + return nil + } + + v4ObsoleteRules := []util.IPTableRule{ + {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)}, + {Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x40000/0x40000`)}, + } + v6ObsoleteRules := []util.IPTableRule{ + {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)}, + {Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x40000/0x40000`)}, + } + + var obsoleteRules []util.IPTableRule + if protocol == kubeovnv1.ProtocolIPv4 { + obsoleteRules = v4ObsoleteRules + } else { + obsoleteRules = v6ObsoleteRules + } + + ipt := c.iptablesObsolete[protocol] + for _, rule := range obsoleteRules { + if err := deleteIptablesRule(ipt, rule); err != nil { + klog.Error(err) + return err + } + } + for _, rule := range rules { + if err := deleteIptablesRule(ipt, rule); err != nil { + klog.Error(err) + return err + } + } + + if err := clearObsoleteIptablesChain(ipt, NAT, OvnPrerouting, Prerouting); err != nil { + return err + } + if err := clearObsoleteIptablesChain(ipt, NAT, OvnPostrouting, Postrouting); err != nil { + return err + } + + delete(c.iptablesObsolete, protocol) + if len(c.iptablesObsolete) == 0 { + c.iptablesObsolete = nil } return nil } diff --git a/pkg/util/strings.go b/pkg/util/strings.go new file mode 100644 index 00000000000..5a1a9ef9843 --- /dev/null +++ b/pkg/util/strings.go @@ -0,0 +1,24 @@ +package util + +import "strings" + +func DoubleQuotedFields(s string) []string { + var quoted bool + var fields []string + sb := &strings.Builder{} + for _, r := range s { + if r == '"' { + quoted = !quoted + } else if !quoted && r == ' ' { + fields = append(fields, sb.String()) + sb.Reset() + } else { + sb.WriteRune(r) + } + } + if sb.Len() > 0 { + fields = append(fields, sb.String()) + } + + return fields +}