diff --git a/go.mod b/go.mod index 90aa3db2e6..f62a1153f2 100644 --- a/go.mod +++ b/go.mod @@ -89,6 +89,7 @@ require ( github.com/Microsoft/hcsshim v0.11.4 github.com/aws/aws-sdk-go v1.45.24 github.com/containerd/continuity v0.4.3 // indirect + github.com/containernetworking/plugins v1.3.0 github.com/google/go-containerregistry v0.19.0 github.com/iamacarpet/go-win64api v0.0.0-20210311141720-fe38760bed28 github.com/k3s-io/helm-controller v0.15.9 @@ -143,6 +144,7 @@ require ( github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/Rican7/retry v0.1.0 // indirect + github.com/alexflint/go-filemutex v1.2.0 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect @@ -174,7 +176,6 @@ require ( github.com/containerd/ttrpc v1.2.3 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/containernetworking/cni v1.1.2 // indirect - github.com/containernetworking/plugins v1.3.0 // indirect github.com/coreos/go-iptables v0.7.0 // indirect github.com/coreos/go-oidc v2.2.1+incompatible // indirect github.com/coreos/go-semver v0.3.1 // indirect diff --git a/go.sum b/go.sum index 9d0dc0f8a6..0bb951ff97 100644 --- a/go.sum +++ b/go.sum @@ -337,6 +337,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/alexflint/go-filemutex v1.2.0 h1:1v0TJPDtlhgpW4nJ+GvxCLSlUDC3+gW0CQQvlmfDR/s= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= diff --git a/pkg/windows/flannel.go b/pkg/windows/flannel.go index 0001751a6f..e93dd0d864 100644 --- a/pkg/windows/flannel.go +++ b/pkg/windows/flannel.go @@ -26,6 +26,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + hostlocaldisk "github.com/containernetworking/plugins/plugins/ipam/host-local/backend/disk" + "k8s.io/utils/pointer" ) @@ -114,19 +116,22 @@ type Flannel struct { } type SourceVipResponse struct { - CniVersion string `json:"cniVersion"` - IPs []struct { - Address string `json:"address"` - Gateway string `json:"gateway"` - } `json:"ips"` - DNS struct{} `json:"dns"` + CniVersion string `json:"cniVersion"` + IPs []struct { + Address string `json:"address"` + Gateway string `json:"gateway"` + } `json:"ips"` + DNS struct{} `json:"dns"` } const ( - FlannelConfigName = "07-flannel.conflist" - FlannelKubeConfigName = "flannel.kubeconfig" - FlanneldConfigName = "flanneld-net-conf.json" - FlannelChart = "rke2-flannel" + flannelConfigName = "07-flannel.conflist" + flannelKubeConfigName = "flannel.kubeconfig" + flanneldConfigName = "flanneld-net-conf.json" + FlannelChart = "rke2-flannel" + hostlocalContainerID = "kube-proxy" + hostlocalInterfaceName = "source-vip" + hostLocalDataDir = "/var/lib/cni/networks" ) // GetConfig returns the CNI configuration @@ -186,17 +191,17 @@ func (f *Flannel) initializeConfig(ctx context.Context, nodeConfig *daemonconfig func (f *Flannel) writeConfigFiles() error { // Create flannelKubeConfig - if err := f.renderFlannelConfig(filepath.Join(f.CNICfg.ConfigPath, FlannelKubeConfigName), flannelKubeConfigTemplate); err != nil { + if err := f.renderFlannelConfig(filepath.Join(f.CNICfg.ConfigPath, flannelKubeConfigName), flannelKubeConfigTemplate); err != nil { return err } // Create flanneld config - if err := f.renderFlannelConfig(filepath.Join(f.CNICfg.ConfigPath, FlanneldConfigName), flanneldConfigTemplate); err != nil { + if err := f.renderFlannelConfig(filepath.Join(f.CNICfg.ConfigPath, flanneldConfigName), flanneldConfigTemplate); err != nil { return err } // Create flannel CNI conflist - if err := f.renderFlannelConfig(filepath.Join(f.CNICfg.CNIConfDir, FlannelConfigName), flannelCniConflistTemplate); err != nil { + if err := f.renderFlannelConfig(filepath.Join(f.CNICfg.CNIConfDir, flannelConfigName), flannelCniConflistTemplate); err != nil { return err } @@ -285,12 +290,12 @@ func startFlannel(ctx context.Context, config *FlannelConfig, logPath string) { } args := []string{ - fmt.Sprintf("--kubeconfig-file=%s", filepath.Join(config.ConfigPath, FlannelKubeConfigName)), + fmt.Sprintf("--kubeconfig-file=%s", filepath.Join(config.ConfigPath, flannelKubeConfigName)), "--ip-masq", "--kube-subnet-mgr", "--iptables-forward-rules=false", fmt.Sprintf("--iface=%s", config.Interface), - fmt.Sprintf("--net-config-path=%s", filepath.Join(config.ConfigPath, FlanneldConfigName)), + fmt.Sprintf("--net-config-path=%s", filepath.Join(config.ConfigPath, flanneldConfigName)), } logrus.Infof("Flanneld Envs: %s and args: %v", specificEnvs, args) @@ -331,26 +336,38 @@ func (f *Flannel) ReserveSourceVip(ctx context.Context) (string, error) { return "", err } + // Check if the source vip was already reserved using host-local library + hostlocalStore, err := hostlocaldisk.New(f.CNICfg.OverlayNetName, hostLocalDataDir) + if err != nil { + return "", fmt.Errorf("failed to create host-local store: %w", err) + } + ips := hostlocalStore.GetByID(hostlocalContainerID, hostlocalInterfaceName) + if len(ips) > 0 { + logrus.Infof("Source VIP for kube-proxy was already reserved %v", ips) + return strings.TrimSpace(strings.Split(ips[0].String(), "/")[0]), nil + } + + logrus.Info("No source VIP for kube-proxy reserved. Creating one") subnet := network.Subnets[0].AddressPrefix logrus.Debugf("host-local will use the following subnet: %v to reserve the sourceIP", subnet) configData := `{ "cniVersion": "1.0.0", - "name": "flannel.4096", + "name": "` + f.CNICfg.OverlayNetName + `", "ipam": { "type": "host-local", "ranges": [[{"subnet":"` + subnet + `"}]], - "dataDir": "/var/lib/cni/networks" + "dataDir": "` + hostLocalDataDir + `" } }` cmd := exec.Command("host-local.exe") cmd.Env = append(os.Environ(), "CNI_COMMAND=ADD", - "CNI_CONTAINERID=kube-proxy", + "CNI_CONTAINERID="+hostlocalContainerID, "CNI_NETNS=kube-proxy", - "CNI_IFNAME=source-vip", + "CNI_IFNAME="+hostlocalInterfaceName, "CNI_PATH="+f.CNICfg.CNIBinDir, ) @@ -373,6 +390,6 @@ func (f *Flannel) ReserveSourceVip(ctx context.Context) (string, error) { if len(sourceVipResp.IPs) > 0 { return strings.TrimSpace(strings.Split(sourceVipResp.IPs[0].Address, "/")[0]), nil } - + return "", errors.New("no source vip reserved") }