diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index e4f5d8d6461..20d4f43da80 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -1887,7 +1887,11 @@ jobs: - name: Install Kube-OVN run: make kind-install - - name: Run E2E + - name: Run Vip E2E + working-directory: ${{ env.E2E_DIR }} + run: make vip-conformance-e2e + + - name: Run Ovn VPC NAT GW E2E working-directory: ${{ env.E2E_DIR }} run: make ovn-vpc-nat-gw-conformance-e2e diff --git a/Makefile b/Makefile index 4291b278d93..c33549e8870 100644 --- a/Makefile +++ b/Makefile @@ -910,7 +910,7 @@ lint: echo "Code differs from gofmt's style" 1>&2 && exit 1; \ fi @GOOS=linux go vet ./... - @GOOS=linux gosec -exclude=G204,G306,G404,G601,G301 -exclude-dir=test -exclude-dir=pkg/client ./... + @GOOS=linux gosec -exclude=G204,G306,G402,G404,G601,G301 -exclude-dir=test -exclude-dir=pkg/client ./... .PHONY: gofumpt gofumpt: gofumpt diff --git a/Makefile.e2e b/Makefile.e2e index 8f142701fed..ac82211f224 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -63,6 +63,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/kube-ovn ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-ic ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ha @@ -133,6 +134,15 @@ kube-ovn-lb-svc-conformance-e2e: ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/lb-svc/lb-svc.test -- $(TEST_BIN_ARGS) +.PHONY: vip-conformance-e2e +vip-conformance-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/vip/vip.test -- $(TEST_BIN_ARGS) + .PHONY: iptables-vpc-nat-gw-conformance-e2e iptables-vpc-nat-gw-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw diff --git a/charts/templates/kube-ovn-crd.yaml b/charts/templates/kube-ovn-crd.yaml index 3c63afcdd5e..ee5b70e8da9 100644 --- a/charts/templates/kube-ovn-crd.yaml +++ b/charts/templates/kube-ovn-crd.yaml @@ -980,8 +980,6 @@ spec: type: string v4Ip: type: string - macAddress: - type: string vpc: type: string conditions: @@ -1010,6 +1008,10 @@ spec: type: string ipName: type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1038,8 +1040,8 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string - - jsonPath: .status.v4ipCidr - name: V4Ip + - jsonPath: .status.v4IpCidr + name: V4IpCidr type: string - jsonPath: .status.ready name: Ready @@ -1055,7 +1057,7 @@ spec: type: boolean v4Eip: type: string - v4ipCidr: + v4IpCidr: type: string vpc: type: string @@ -1085,6 +1087,10 @@ spec: type: string ipName: type: string + vpc: + type: string + v4IpCidr: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1107,6 +1113,9 @@ spec: subresources: status: {} additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string - jsonPath: .spec.ovnEip name: Eip type: string @@ -1144,8 +1153,6 @@ spec: type: string v4Ip: type: string - macAddress: - type: string vpc: type: string externalPort: @@ -1188,6 +1195,10 @@ spec: type: string protocol: type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index c7b79e3b8a0..3887715cab1 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -47,11 +47,11 @@ func dumpProfile() { klog.Errorf("failed to create cpu profile file: %v", err) return } - defer f.Close() if err = pprof.StartCPUProfile(f); err != nil { klog.Errorf("failed to start cpu profile: %v", err) return } + defer f.Close() time.Sleep(30 * time.Second) pprof.StopCPUProfile() } @@ -65,10 +65,10 @@ func dumpProfile() { klog.Errorf("failed to create memory profile file: %v", err) return } - defer f.Close() if err = pprof.WriteHeapProfile(f); err != nil { klog.Errorf("failed to write memory profile file: %v", err) } + defer f.Close() } }() } diff --git a/dist/images/install.sh b/dist/images/install.sh index 42ddeffc534..1c30498ac6b 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1519,8 +1519,6 @@ spec: type: string v4Ip: type: string - macAddress: - type: string vpc: type: string conditions: @@ -1549,6 +1547,10 @@ spec: type: string ipName: type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1577,8 +1579,8 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string - - jsonPath: .status.v4ipCidr - name: V4Ip + - jsonPath: .status.v4IpCidr + name: V4IpCidr type: string - jsonPath: .status.ready name: Ready @@ -1594,7 +1596,7 @@ spec: type: boolean v4Eip: type: string - v4ipCidr: + v4IpCidr: type: string vpc: type: string @@ -1624,6 +1626,10 @@ spec: type: string ipName: type: string + vpc: + type: string + v4IpCidr: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1646,6 +1652,9 @@ spec: subresources: status: {} additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string - jsonPath: .spec.ovnEip name: Eip type: string @@ -1683,8 +1692,6 @@ spec: type: string v4Ip: type: string - macAddress: - type: string vpc: type: string externalPort: @@ -1727,6 +1734,10 @@ spec: type: string protocol: type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/go.mod b/go.mod index 0fc6fb733de..ef292775cfd 100644 --- a/go.mod +++ b/go.mod @@ -42,17 +42,17 @@ require ( google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.0.2 - k8s.io/api v0.28.2 - k8s.io/apimachinery v0.28.2 + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 k8s.io/client-go v12.0.0+incompatible k8s.io/klog/v2 v2.100.1 k8s.io/kubectl v0.28.2 k8s.io/kubernetes v1.28.2 k8s.io/pod-security-admission v0.28.2 - k8s.io/sample-controller v0.28.2 + k8s.io/sample-controller v0.28.3 k8s.io/utils v0.0.0-20230726121419-3b25d923346b kubevirt.io/client-go v1.0.0 - sigs.k8s.io/controller-runtime v0.16.2 + sigs.k8s.io/controller-runtime v0.16.3 ) require ( @@ -250,18 +250,18 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.2 // indirect - k8s.io/apiserver v0.28.2 // indirect + k8s.io/apiextensions-apiserver v0.28.3 // indirect + k8s.io/apiserver v0.28.3 // indirect k8s.io/cli-runtime v0.28.2 // indirect k8s.io/cloud-provider v0.28.2 // indirect k8s.io/cluster-bootstrap v0.28.2 // indirect - k8s.io/component-base v0.28.2 // indirect + k8s.io/component-base v0.28.3 // indirect k8s.io/component-helpers v0.28.2 // indirect k8s.io/controller-manager v0.28.2 // indirect k8s.io/cri-api v0.28.2 // indirect k8s.io/csi-translation-lib v0.28.2 // indirect k8s.io/dynamic-resource-allocation v0.0.0 // indirect - k8s.io/kms v0.28.2 // indirect + k8s.io/kms v0.28.3 // indirect k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect k8s.io/kube-scheduler v0.0.0 // indirect k8s.io/kubelet v0.28.2 // indirect diff --git a/go.sum b/go.sum index 21fab8e7561..e609527aabb 100644 --- a/go.sum +++ b/go.sum @@ -3049,8 +3049,9 @@ k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.28.2 h1:KhG63LHopCdzs1oKA1j+NWleuIXudgOyCqJo4yi3GaM= k8s.io/kms v0.28.2/go.mod h1:iAjgIqBrV2+8kmsjbbgUkAyKSuYq5g1dW9knpt6OhaE= +k8s.io/kms v0.28.3 h1:jYwwAe96XELNjYWv1G4kNzizcFoZ50OOElvPansbw70= +k8s.io/kms v0.28.3/go.mod h1:kSMjU2tg7vjqqoWVVCcmPmNZ/CofPsoTbSxAipCvZuE= k8s.io/kube-aggregator v0.28.2 h1:tCjAfB1p/v18yD2NpegNQRuahzyA/szFfcRARnpjDeo= k8s.io/kube-aggregator v0.28.2/go.mod h1:g4hZVjC4KhJtZHV2pyiRBiU6AdBA/sAjh9Y9GJC/SbU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= @@ -3076,8 +3077,8 @@ k8s.io/pod-security-admission v0.28.2 h1:3kiOL+gc6auNTGHuQ0hVsGxYu2YO/7DZb0xYR84 k8s.io/pod-security-admission v0.28.2/go.mod h1:gReea39xbhIzf4Ry0FDuiTi8uj1N5R9YXOh8zQSuTxs= k8s.io/sample-apiserver v0.28.2 h1:lT4EeLgGe0yxZhWlXmUBzWtqUxUc1X+vO8PmIYuFOVw= k8s.io/sample-apiserver v0.28.2/go.mod h1:xLogWmyVOkxCxCTu+NlwucWi/yxb/VZlmDWEurF5rOs= -k8s.io/sample-controller v0.28.2 h1:zIoifZJkqCitdqLdI06Uh4LOY9/w92vImSOydOMCNDI= -k8s.io/sample-controller v0.28.2/go.mod h1:QoJ32+bl8u2nZIPG55ZzZGO/UAuKAHmaw48mTXgjQ18= +k8s.io/sample-controller v0.28.3 h1:yS3hNgOFpgaIldluEAlSLIRk2281g07kxNrdwKBX2gM= +k8s.io/sample-controller v0.28.3/go.mod h1:R6ahxfGNbfqHpCF53ogmYoIn80jQrBNvEjKCGAYufHQ= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -3134,8 +3135,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= -sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= -sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index a02ae615c2f..381c46e5787 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -1003,6 +1003,8 @@ type OvnFipSpec struct { OvnEip string `json:"ovnEip"` IPType string `json:"ipType"` // vip, ip IPName string `json:"ipName"` // vip, ip crd name + Vpc string `json:"vpc"` + V4Ip string `json:"v4Ip"` } // OvnFipCondition describes the state of an object at a certain point. @@ -1012,11 +1014,10 @@ type OvnFipCondition Condition type OvnFipStatus struct { // +optional // +patchStrategy=merge - Ready bool `json:"ready" patchStrategy:"merge"` - V4Eip string `json:"v4Eip" patchStrategy:"merge"` - V4Ip string `json:"v4Ip" patchStrategy:"merge"` - MacAddress string `json:"macAddress" patchStrategy:"merge"` - Vpc string `json:"vpc" patchStrategy:"merge"` + Vpc string `json:"vpc" patchStrategy:"merge"` + V4Eip string `json:"v4Eip" patchStrategy:"merge"` + V4Ip string `json:"v4Ip" patchStrategy:"merge"` + Ready bool `json:"ready" patchStrategy:"merge"` // Conditions represents the latest state of the object // +optional @@ -1051,6 +1052,8 @@ type OvnSnatRuleSpec struct { OvnEip string `json:"ovnEip"` VpcSubnet string `json:"vpcSubnet"` IPName string `json:"ipName"` + Vpc string `json:"vpc"` + V4IpCidr string `json:"v4IpCidr"` // subnet cidr or pod ip address } // OvnSnatRuleCondition describes the state of an object at a certain point. @@ -1060,10 +1063,10 @@ type OvnSnatRuleCondition Condition type OvnSnatRuleStatus struct { // +optional // +patchStrategy=merge - Ready bool `json:"ready" patchStrategy:"merge"` - V4Eip string `json:"v4Eip" patchStrategy:"merge"` - V4IpCidr string `json:"v4ipCidr" patchStrategy:"merge"` Vpc string `json:"vpc" patchStrategy:"merge"` + V4Eip string `json:"v4Eip" patchStrategy:"merge"` + V4IpCidr string `json:"v4IpCidr" patchStrategy:"merge"` + Ready bool `json:"ready" patchStrategy:"merge"` // Conditions represents the latest state of the object // +optional @@ -1101,6 +1104,8 @@ type OvnDnatRuleSpec struct { InternalPort string `json:"internalPort"` ExternalPort string `json:"externalPort"` Protocol string `json:"protocol,omitempty"` + Vpc string `json:"vpc"` + V4Ip string `json:"v4Ip"` } // OvnDnatRuleCondition describes the state of an object at a certain point. @@ -1111,15 +1116,14 @@ type OvnDnatRuleCondition Condition type OvnDnatRuleStatus struct { // +optional // +patchStrategy=merge - Ready bool `json:"ready" patchStrategy:"merge"` + Vpc string `json:"vpc" patchStrategy:"merge"` V4Eip string `json:"v4Eip" patchStrategy:"merge"` + ExternalPort string `json:"externalPort"` V4Ip string `json:"v4Ip" patchStrategy:"merge"` - MacAddress string `json:"macAddress" patchStrategy:"merge"` - Vpc string `json:"vpc" patchStrategy:"merge"` InternalPort string `json:"internalPort"` - ExternalPort string `json:"externalPort"` Protocol string `json:"protocol,omitempty"` IPName string `json:"ipName"` + Ready bool `json:"ready" patchStrategy:"merge"` // Conditions represents the latest state of the object // +optional diff --git a/pkg/controller/ovn_dnat.go b/pkg/controller/ovn_dnat.go index c9833a4f3a2..687a560df1d 100644 --- a/pkg/controller/ovn_dnat.go +++ b/pkg/controller/ovn_dnat.go @@ -185,7 +185,7 @@ func (c *Controller) processNextDeleteOvnDnatRuleWorkItem() bool { return true } -func (c *Controller) isOvnDnatDuplicated(eipName, dnatName, externalPort string) (bool, error) { +func (c *Controller) isOvnDnatDuplicated(eipName, dnatName, externalPort string) error { // check if eip:external port already used dnats, err := c.ovnDnatRulesLister.List(labels.SelectorFromSet(labels.Set{ util.VpcDnatEPortLabel: externalPort, @@ -193,18 +193,18 @@ func (c *Controller) isOvnDnatDuplicated(eipName, dnatName, externalPort string) if err != nil { if !k8serrors.IsNotFound(err) { klog.Error(err) - return false, err + return err } } if len(dnats) != 0 { for _, d := range dnats { if d.Name != dnatName && d.Spec.OvnEip == eipName { err = fmt.Errorf("failed to create dnat %s, duplicate, same eip %s, same external port '%s' is using by dnat %s", dnatName, eipName, externalPort, d.Name) - return true, err + return err } } } - return false, nil + return nil } func (c *Controller) handleAddOvnDnatRule(key string) error { @@ -222,68 +222,75 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { return nil } klog.Infof("handle add dnat %s", key) - var internalV4Ip, mac, subnetName string - if cachedDnat.Spec.IPType == util.Vip { - internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IPName) - if err != nil { - klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IPName, err) - return err - } - internalV4Ip = internalVip.Status.V4ip - mac = internalVip.Status.Mac - subnetName = internalVip.Spec.Subnet - } else { - internalIP, err := c.ipsLister.Get(cachedDnat.Spec.IPName) - if err != nil { - klog.Errorf("failed to get ip %s, %v", cachedDnat.Spec.IPName, err) - return err - } - internalV4Ip = internalIP.Spec.V4IPAddress - mac = internalIP.Spec.MacAddress - subnetName = internalIP.Spec.Subnet - } - + // check eip eipName := cachedDnat.Spec.OvnEip - if len(eipName) == 0 { + if eipName == "" { err := fmt.Errorf("failed to create dnat %s, should set eip", cachedDnat.Name) klog.Error(err) return err } - cachedEip, err := c.GetOvnEip(eipName) if err != nil { klog.Errorf("failed to get eip, %v", err) return err } - if cachedEip.Spec.Type == util.Lsp { // eip is using by ecmp nexthop lsp, nat can not use err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) klog.Error(err) return err } - - if _, err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { - klog.Error("failed to create dnat %s, %v", cachedDnat.Name, err) + if cachedEip.Status.V4Ip == "" { + err := fmt.Errorf("failed to create v4 dnat %s, eip %s has no v4 ip", cachedDnat.Name, eipName) + klog.Error(err) return err } - - subnet, err := c.subnetsLister.Get(subnetName) - if err != nil { - klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) + if err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { + klog.Error("failed to create dnat %s, %v", cachedDnat.Name, err) return err } - if cachedEip.Status.V4Ip == "" || internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 dnat %s", cachedDnat.Name) + // get dnat external ip, internal ip, vpc name + var internalV4Ip, subnetName, vpcName string + if cachedDnat.Spec.Vpc != "" { + vpcName = cachedDnat.Spec.Vpc + } + if cachedDnat.Spec.V4Ip != "" { + internalV4Ip = cachedDnat.Spec.V4Ip + } + if internalV4Ip == "" && cachedDnat.Spec.IPName != "" { + if cachedDnat.Spec.IPType == util.Vip { + internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IPName) + if err != nil { + klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IPName, err) + return err + } + internalV4Ip = internalVip.Status.V4ip + subnetName = internalVip.Spec.Subnet + } else { + internalIP, err := c.ipsLister.Get(cachedDnat.Spec.IPName) + if err != nil { + klog.Errorf("failed to get ip %s, %v", cachedDnat.Spec.IPName, err) + return err + } + internalV4Ip = internalIP.Spec.V4IPAddress + subnetName = internalIP.Spec.Subnet + } + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) + return err + } + vpcName = subnet.Spec.Vpc + } + if internalV4Ip == "" { + err := fmt.Errorf("failed to create v4 dnat %s, no internal v4 ip", cachedDnat.Name) klog.Error(err) return err } - - vpcName := subnet.Spec.Vpc - if err = c.patchOvnDnatStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, mac, false); err != nil { - klog.Errorf("failed to patch status for dnat %s, %v", key, err) + if vpcName == "" { + err := fmt.Errorf("failed to create v4 dnat %s, no vpc", cachedDnat.Name) + klog.Error(err) return err } @@ -315,7 +322,7 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { } if err = c.patchOvnDnatStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, mac, true); err != nil { + internalV4Ip, true); err != nil { klog.Errorf("failed to patch status for dnat %s, %v", key, err) return err } @@ -366,66 +373,74 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { } klog.Infof("handle update dnat %s", key) - var internalV4Ip, mac, subnetName string - if cachedDnat.Spec.IPType == util.Vip { - internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IPName) - if err != nil { - klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IPName, err) - return err - } - internalV4Ip = internalVip.Status.V4ip - mac = internalVip.Status.Mac - subnetName = internalVip.Spec.Subnet - } else { - internalIP, err := c.ipsLister.Get(cachedDnat.Spec.IPName) - if err != nil { - klog.Errorf("failed to get ip %s, %v", cachedDnat.Spec.IPName, err) - return err - } - internalV4Ip = internalIP.Spec.V4IPAddress - mac = internalIP.Spec.MacAddress - subnetName = internalIP.Spec.Subnet - } - + // check eip eipName := cachedDnat.Spec.OvnEip - if len(eipName) == 0 { + if eipName == "" { err := fmt.Errorf("failed to create dnat %s, should set eip", cachedDnat.Name) klog.Error(err) return err } - cachedEip, err := c.GetOvnEip(eipName) if err != nil { klog.Errorf("failed to get eip, %v", err) return err } - - if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip { - err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip) + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) return err } - - if _, err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { - klog.Error("failed to update dnat %s, %v", cachedDnat.Name, err) + if cachedEip.Status.V4Ip == "" { + err := fmt.Errorf("failed to create v4 dnat %s, eip %s has no v4 ip", cachedDnat.Name, eipName) + klog.Error(err) return err } - - subnet, err := c.subnetsLister.Get(subnetName) - if err != nil { - klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) + if err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { + klog.Error("failed to create dnat %s, %v", cachedDnat.Name, err) return err } - vpcName := subnet.Spec.Vpc - if cachedEip.Status.V4Ip == "" || internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 dnat %s", cachedDnat.Name) + // get dnat external ip, internal ip, vpc name + var internalV4Ip, subnetName, vpcName string + if cachedDnat.Spec.Vpc != "" { + vpcName = cachedDnat.Spec.Vpc + } + if cachedDnat.Spec.V4Ip != "" { + internalV4Ip = cachedDnat.Spec.V4Ip + } + if internalV4Ip == "" && cachedDnat.Spec.IPName != "" { + if cachedDnat.Spec.IPType == util.Vip { + internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IPName) + if err != nil { + klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IPName, err) + return err + } + internalV4Ip = internalVip.Status.V4ip + subnetName = internalVip.Spec.Subnet + } else { + internalIP, err := c.ipsLister.Get(cachedDnat.Spec.IPName) + if err != nil { + klog.Errorf("failed to get ip %s, %v", cachedDnat.Spec.IPName, err) + return err + } + internalV4Ip = internalIP.Spec.V4IPAddress + subnetName = internalIP.Spec.Subnet + } + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) + return err + } + vpcName = subnet.Spec.Vpc + } + if internalV4Ip == "" { + err := fmt.Errorf("failed to create v4 dnat %s, no internal v4 ip", cachedDnat.Name) klog.Error(err) return err } - - if cachedEip.Spec.Type == util.Lsp { - // eip is using by ecmp nexthop lsp, nat can not use - err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + if vpcName == "" { + err := fmt.Errorf("failed to create v4 dnat %s, no vpc", cachedDnat.Name) klog.Error(err) return err } @@ -454,7 +469,7 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { return err } - if err = c.patchOvnDnatStatus(key, vpcName, cachedEip.Status.V4Ip, internalV4Ip, mac, true); err != nil { + if err = c.patchOvnDnatStatus(key, vpcName, cachedEip.Status.V4Ip, internalV4Ip, true); err != nil { klog.Errorf("failed to patch status for dnat '%s', %v", key, err) return err } @@ -508,12 +523,11 @@ func (c *Controller) patchOvnDnatAnnotations(key, eipName string) error { return nil } -func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP, podMac string, ready bool) error { +func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP string, ready bool) error { var ( oriDnat, dnat *kubeovnv1.OvnDnatRule err error ) - if oriDnat, err = c.ovnDnatRulesLister.Get(key); err != nil { if k8serrors.IsNotFound(err) { return nil @@ -522,7 +536,6 @@ func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP, podMac strin return err } dnat = oriDnat.DeepCopy() - var ( needUpdateLabel = false changed bool @@ -556,14 +569,18 @@ func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP, podMac strin changed = true } - if (v4Eip != "" && dnat.Status.V4Eip != v4Eip) || - (vpcName != "" && dnat.Status.Vpc != vpcName) || - (podIP != "" && dnat.Status.V4Ip != podIP) || - (podMac != "" && dnat.Status.MacAddress != podMac) { + if vpcName != "" && dnat.Status.Vpc != vpcName { dnat.Status.Vpc = vpcName + changed = true + } + + if v4Eip != "" && dnat.Status.V4Eip != v4Eip { dnat.Status.V4Eip = v4Eip + changed = true + } + + if podIP != "" && dnat.Status.V4Ip != podIP { dnat.Status.V4Ip = podIP - dnat.Status.MacAddress = podMac changed = true } diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 62026af63cf..73d87d28d65 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -223,7 +223,9 @@ func (c *Controller) handleAddOvnEip(key string) error { var v4ip, v6ip, mac, subnetName string subnetName = cachedEip.Spec.ExternalSubnet if subnetName == "" { - return fmt.Errorf("failed to create ovn eip '%s', subnet should be set", key) + err := fmt.Errorf("failed to create ovn eip '%s', subnet should be set", key) + klog.Error(err) + return err } subnet, err := c.subnetsLister.Get(subnetName) if err != nil { @@ -284,7 +286,9 @@ func (c *Controller) handleUpdateOvnEip(key string) error { if !cachedEip.DeletionTimestamp.IsZero() { subnetName := cachedEip.Spec.ExternalSubnet if subnetName == "" { - return fmt.Errorf("failed to update ovn eip '%s', subnet should be set", key) + err := fmt.Errorf("failed to create ovn eip '%s', subnet should be set", key) + klog.Error(err) + return err } subnet, err := c.subnetsLister.Get(subnetName) if err != nil { @@ -394,25 +398,49 @@ func (c *Controller) createOrUpdateCrdOvnEip(key, subnet, v4ip, v6ip, mac, usage } } else { ovnEip := cachedEip.DeepCopy() - if ovnEip.Spec.V4Ip == "" && v4ip != "" || - ovnEip.Spec.V6Ip == "" && v6ip != "" { - ovnEip.Spec.ExternalSubnet = subnet + needUpdate := false + + if mac != "" && ovnEip.Spec.MacAddress != mac { + ovnEip.Spec.MacAddress = mac + needUpdate = true + } + if v4ip != "" && ovnEip.Spec.V4Ip != v4ip { ovnEip.Spec.V4Ip = v4ip + needUpdate = true + } + if v6ip != "" && ovnEip.Spec.V6Ip != v6ip { ovnEip.Spec.V6Ip = v6ip - ovnEip.Spec.MacAddress = mac + needUpdate = true + } + if usage != "" && ovnEip.Spec.Type != usage { ovnEip.Spec.Type = usage + needUpdate = true + } + if needUpdate { if _, err := c.config.KubeOvnClient.KubeovnV1().OvnEips().Update(context.Background(), ovnEip, metav1.UpdateOptions{}); err != nil { errMsg := fmt.Errorf("failed to update ovn eip '%s', %v", key, err) klog.Error(errMsg) return errMsg } } - - if ovnEip.Status.MacAddress == "" { + needPatch := false + if ovnEip.Status.V4Ip == "" && ovnEip.Status.V4Ip != v4ip { ovnEip.Status.V4Ip = v4ip + needPatch = true + } + if ovnEip.Status.V6Ip == "" && ovnEip.Status.V6Ip != v6ip { ovnEip.Status.V6Ip = v6ip + needPatch = true + } + if ovnEip.Status.MacAddress == "" && ovnEip.Status.MacAddress != mac { ovnEip.Status.MacAddress = mac + needPatch = true + } + if ovnEip.Status.Type == "" && ovnEip.Status.Type != usage { ovnEip.Status.Type = usage + needPatch = true + } + if needPatch { bytes, err := ovnEip.Status.Bytes() if err != nil { klog.Error("failed to marshal ovn eip %s, %v", key, err) diff --git a/pkg/controller/ovn_fip.go b/pkg/controller/ovn_fip.go index b35a71d2985..83dc4760872 100644 --- a/pkg/controller/ovn_fip.go +++ b/pkg/controller/ovn_fip.go @@ -174,7 +174,7 @@ func (c *Controller) processNextDeleteOvnFipWorkItem() bool { return true } -func (c *Controller) ovnFipTryUseEip(fipName, eipV4IP string) error { +func (c *Controller) isOvnFipDuplicated(fipName, eipV4IP string) error { // check if has another fip using this eip already selector := labels.SelectorFromSet(labels.Set{util.EipV4IpLabel: eipV4IP}) usingFips, err := c.ovnFipsLister.List(selector) @@ -206,30 +206,9 @@ func (c *Controller) handleAddOvnFip(key string) error { return nil } klog.Infof("handle add fip %s", key) - var internalV4Ip, mac, subnetName string - if cachedFip.Spec.IPType == util.Vip { - internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IPName) - if err != nil { - klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IPName, err) - return err - } - internalV4Ip = internalVip.Status.V4ip - mac = internalVip.Status.Mac - subnetName = internalVip.Spec.Subnet - } else { - internalIP, err := c.ipsLister.Get(cachedFip.Spec.IPName) - if err != nil { - klog.Errorf("failed to get ip %s, %v", cachedFip.Spec.IPName, err) - return err - } - internalV4Ip = internalIP.Spec.V4IPAddress - mac = internalIP.Spec.MacAddress - subnetName = internalIP.Spec.Subnet - } - - // get eip + // check eip eipName := cachedFip.Spec.OvnEip - if len(eipName) == 0 { + if eipName == "" { err := fmt.Errorf("failed to create fip rule, should set eip") klog.Error(err) return err @@ -239,44 +218,74 @@ func (c *Controller) handleAddOvnFip(key string) error { klog.Errorf("failed to get eip, %v", err) return err } - if cachedEip.Spec.Type == util.Lsp { // eip is using by ecmp nexthop lsp, nat can not use err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) klog.Error(err) return err } - - if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil { - err = fmt.Errorf("failed to add fip %s, %v", key, err) + if cachedEip.Status.V4Ip == "" { + err := fmt.Errorf("failed to create v4 fip %s, eip %s has no v4 ip", cachedFip.Name, eipName) klog.Error(err) return err } - - subnet, err := c.subnetsLister.Get(subnetName) - if err != nil { - klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) - return err - } - if cachedEip.Status.V4Ip == "" || internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 fip %s", cachedFip.Name) + if err = c.isOvnFipDuplicated(key, cachedEip.Spec.V4Ip); err != nil { + err = fmt.Errorf("failed to add fip %s, %v", key, err) klog.Error(err) return err } - vpcName := subnet.Spec.Vpc - if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip { - err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip) - return err - } - if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil { - err = fmt.Errorf("failed to update fip %s, %v", key, err) + var mac, internalV4Ip, subnetName, vpcName string + if cachedFip.Spec.Vpc != "" { + vpcName = cachedFip.Spec.Vpc + } + if cachedFip.Spec.V4Ip != "" { + internalV4Ip = cachedFip.Spec.V4Ip + } + if internalV4Ip == "" && cachedFip.Spec.IPName != "" { + if cachedFip.Spec.IPType == util.Vip { + internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IPName) + if err != nil { + klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IPName, err) + return err + } + internalV4Ip = internalVip.Status.V4ip + subnetName = internalVip.Spec.Subnet + // though vip lsp has its mac, vip always use its parent lsp nic mac + // and vip could float to different parent lsp nic + // all vip its parent lsp acl should allow the vip ip + } else { + internalIP, err := c.ipsLister.Get(cachedFip.Spec.IPName) + if err != nil { + klog.Errorf("failed to get ip %s, %v", cachedFip.Spec.IPName, err) + return err + } + internalV4Ip = internalIP.Spec.V4IPAddress + subnetName = internalIP.Spec.Subnet + mac = internalIP.Spec.MacAddress + // mac is necessary while using distributed router fip, fip use lsp its mac + // centralized router fip not need lsp mac, fip use lrp mac + } + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) + return err + } + vpcName = subnet.Spec.Vpc + if err = c.isOvnFipDuplicated(key, cachedEip.Spec.V4Ip); err != nil { + err = fmt.Errorf("failed to update fip %s, %v", key, err) + klog.Error(err) + return err + } + } + if internalV4Ip == "" { + err := fmt.Errorf("failed to create v4 fip %s, no internal v4 ip", cachedFip.Name) klog.Error(err) return err } - if err = c.patchOvnFipStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, mac, false); err != nil { - klog.Errorf("failed to patch status for fip %s, %v", key, err) + if vpcName == "" { + err := fmt.Errorf("failed to create v4 fip %s, no vpc", cachedFip.Name) + klog.Error(err) return err } if err = c.handleAddOvnEipFinalizer(cachedEip, util.ControllerName); err != nil { @@ -284,9 +293,9 @@ func (c *Controller) handleAddOvnFip(key string) error { return err } // ovn add fip + options := map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)} if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip, - internalV4Ip, mac, cachedFip.Spec.IPName, - map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)}); err != nil { + internalV4Ip, mac, cachedFip.Spec.IPName, options); err != nil { klog.Errorf("failed to create v4 fip, %v", err) return err } @@ -306,7 +315,7 @@ func (c *Controller) handleAddOvnFip(key string) error { return err } if err = c.patchOvnFipStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, mac, true); err != nil { + internalV4Ip, true); err != nil { klog.Errorf("failed to patch status for fip %s, %v", key, err) return err } @@ -326,29 +335,9 @@ func (c *Controller) handleUpdateOvnFip(key string) error { return err } klog.Infof("handle update fip %s", key) - var internalV4Ip, mac, subnetName string - if cachedFip.Spec.IPType == util.Vip { - internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IPName) - if err != nil { - klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IPName, err) - return err - } - internalV4Ip = internalVip.Status.V4ip - mac = internalVip.Status.Mac - subnetName = internalVip.Spec.Subnet - } else { - internalIP, err := c.ipsLister.Get(cachedFip.Spec.IPName) - if err != nil { - klog.Errorf("failed to get ip %s, %v", cachedFip.Spec.IPName, err) - return err - } - internalV4Ip = internalIP.Spec.V4IPAddress - mac = internalIP.Spec.MacAddress - subnetName = internalIP.Spec.Subnet - } - // get eip + // check eip eipName := cachedFip.Spec.OvnEip - if len(eipName) == 0 { + if eipName == "" { err := fmt.Errorf("failed to create fip rule, should set eip") klog.Error(err) return err @@ -364,26 +353,72 @@ func (c *Controller) handleUpdateOvnFip(key string) error { klog.Error(err) return err } - if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil { - err = fmt.Errorf("failed to update fip %s, %v", key, err) + if cachedEip.Status.V4Ip == "" { + err := fmt.Errorf("failed to create v4 fip %s, eip %s has no v4 ip", cachedFip.Name, eipName) klog.Error(err) return err } - subnet, err := c.subnetsLister.Get(subnetName) - if err != nil { - klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) + if err = c.isOvnFipDuplicated(key, cachedEip.Spec.V4Ip); err != nil { + err = fmt.Errorf("failed to add fip %s, %v", key, err) + klog.Error(err) return err } - if cachedEip.Status.V4Ip == "" || internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 fip %s", cachedFip.Name) + + var mac, internalV4Ip, subnetName, vpcName string + if cachedFip.Spec.Vpc != "" { + vpcName = cachedFip.Spec.Vpc + } + if cachedFip.Spec.V4Ip != "" { + internalV4Ip = cachedFip.Spec.V4Ip + } + if internalV4Ip == "" && cachedFip.Spec.IPName != "" { + if cachedFip.Spec.IPType == util.Vip { + internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IPName) + if err != nil { + klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IPName, err) + return err + } + internalV4Ip = internalVip.Status.V4ip + subnetName = internalVip.Spec.Subnet + // though vip lsp has its mac, vip always use its parent lsp nic mac + // and vip could float to different parent lsp nic + // all vip its parent lsp acl should allow the vip ip + } else { + internalIP, err := c.ipsLister.Get(cachedFip.Spec.IPName) + if err != nil { + klog.Errorf("failed to get ip %s, %v", cachedFip.Spec.IPName, err) + return err + } + internalV4Ip = internalIP.Spec.V4IPAddress + subnetName = internalIP.Spec.Subnet + mac = internalIP.Spec.MacAddress + // mac is necessary while using distributed router fip, fip use lsp its mac + // centralized router fip not need lsp mac, fip use lrp mac + } + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) + return err + } + vpcName = subnet.Spec.Vpc + if err = c.isOvnFipDuplicated(key, cachedEip.Spec.V4Ip); err != nil { + err = fmt.Errorf("failed to update fip %s, %v", key, err) + klog.Error(err) + return err + } + } + if internalV4Ip == "" { + err := fmt.Errorf("failed to create v4 fip %s, no internal v4 ip", cachedFip.Name) klog.Error(err) return err } - vpcName := subnet.Spec.Vpc - if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip, - internalV4Ip, mac, cachedFip.Spec.IPName, - map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)}); err != nil { - klog.Errorf("failed to create v4 fip, %v", err) + if vpcName == "" { + err := fmt.Errorf("failed to create v4 fip %s, no vpc", cachedFip.Name) + klog.Error(err) + return err + } + if err = c.handleAddOvnEipFinalizer(cachedEip, util.ControllerName); err != nil { + klog.Errorf("failed to add finalizer for ovn eip, %v", err) return err } fip := cachedFip.DeepCopy() @@ -395,9 +430,9 @@ func (c *Controller) handleUpdateOvnFip(key string) error { return err } // ovn add fip + options := map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)} if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip, - internalV4Ip, mac, cachedFip.Spec.IPName, - map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)}); err != nil { + internalV4Ip, mac, cachedFip.Spec.IPName, options); err != nil { klog.Errorf("failed to create fip, %v", err) return err } @@ -410,7 +445,7 @@ func (c *Controller) handleUpdateOvnFip(key string) error { return err } if err = c.patchOvnFipStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, mac, true); err != nil { + internalV4Ip, true); err != nil { klog.Errorf("failed to patch status for fip '%s', %v", key, err) return err } @@ -484,7 +519,7 @@ func (c *Controller) patchOvnFipAnnotations(key, eipName string) error { return nil } -func (c *Controller) patchOvnFipStatus(key, vpcName, v4Eip, podIP, podMac string, ready bool) error { +func (c *Controller) patchOvnFipStatus(key, vpcName, v4Eip, podIP string, ready bool) error { oriFip, err := c.ovnFipsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { @@ -522,14 +557,16 @@ func (c *Controller) patchOvnFipStatus(key, vpcName, v4Eip, podIP, podMac string fip.Status.Ready = ready changed = true } - if (v4Eip != "" && fip.Status.V4Eip != v4Eip) || - (vpcName != "" && fip.Status.Vpc != vpcName) || - (podIP != "" && fip.Status.V4Ip != podIP) || - (podMac != "" && fip.Status.MacAddress != podMac) { + if vpcName != "" && fip.Status.Vpc != vpcName { fip.Status.Vpc = vpcName + changed = true + } + if v4Eip != "" && fip.Status.V4Eip != v4Eip { fip.Status.V4Eip = v4Eip + changed = true + } + if podIP != "" && fip.Status.V4Ip != podIP { fip.Status.V4Ip = podIP - fip.Status.MacAddress = podMac changed = true } if changed { diff --git a/pkg/controller/ovn_snat.go b/pkg/controller/ovn_snat.go index 86a1ca683b8..ec22aad3367 100644 --- a/pkg/controller/ovn_snat.go +++ b/pkg/controller/ovn_snat.go @@ -186,9 +186,10 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { return nil } klog.Infof("handle add ovn snat %s", key) + // check eip eipName := cachedSnat.Spec.OvnEip - if len(eipName) == 0 { - err := fmt.Errorf("failed to create fip rule, should set eip") + if eipName == "" { + err := fmt.Errorf("failed to create ovn snat rule, should set eip") klog.Error(err) return err } @@ -197,16 +198,25 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { klog.Errorf("failed to get eip, %v", err) return err } - if cachedEip.Spec.Type == util.Lsp { // eip is using by ecmp nexthop lsp, nat can not use err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) klog.Error(err) return err } - + if cachedEip.Status.V4Ip == "" { + err := fmt.Errorf("failed to create v4 snat %s, eip %s has no v4 ip", cachedSnat.Name, eipName) + klog.Error(err) + return err + } var v4IpCidr, vpcName string - if cachedSnat.Spec.VpcSubnet != "" { + if cachedSnat.Spec.Vpc != "" { + vpcName = cachedSnat.Spec.Vpc + } + if cachedSnat.Spec.V4IpCidr != "" { + v4IpCidr = cachedSnat.Spec.V4IpCidr + } + if v4IpCidr == "" && cachedSnat.Spec.VpcSubnet != "" { subnet, err := c.subnetsLister.Get(cachedSnat.Spec.VpcSubnet) if err != nil { klog.Errorf("failed to get vpc subnet %s, %v", cachedSnat.Spec.VpcSubnet, err) @@ -215,7 +225,7 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { vpcName = subnet.Spec.Vpc v4IpCidr = subnet.Spec.CIDRBlock } - if cachedSnat.Spec.IPName != "" { + if v4IpCidr == "" && cachedSnat.Spec.IPName != "" { vpcPodIP, err := c.ipsLister.Get(cachedSnat.Spec.IPName) if err != nil { klog.Errorf("failed to get pod ip %s, %v", cachedSnat.Spec.IPName, err) @@ -229,15 +239,15 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { vpcName = subnet.Spec.Vpc v4IpCidr = vpcPodIP.Spec.V4IPAddress } - if v4IpCidr == "" { // only support IPv4 snat err = fmt.Errorf("failed to get v4 internal ip for snat %s", key) + klog.Error(err) return err } - - if err = c.patchOvnSnatStatus(key, vpcName, cachedEip.Spec.V4Ip, v4IpCidr, false); err != nil { - klog.Errorf("failed to update status for snat %s, %v", key, err) + if vpcName == "" { + err := fmt.Errorf("failed to get vpc for snat %s", cachedSnat.Name) + klog.Error(err) return err } @@ -246,7 +256,7 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { klog.Errorf("failed to add finalizer for ovn eip, %v", err) return err } - // ovn add snat + // about conflicts: if multi vpc snat use the same eip, if only one gw node exist, it may should work if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeSNAT, cachedEip.Spec.V4Ip, v4IpCidr, "", "", nil); err != nil { klog.Errorf("failed to create snat, %v", err) return err @@ -283,42 +293,50 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { klog.Error(err) return err } - klog.Infof("handle update ovn snat %s", key) - eipName := cachedSnat.Spec.OvnEip - if len(eipName) == 0 { - err := fmt.Errorf("failed to create fip rule, should set eip") - klog.Error(err) - return err - } - cachedEip, err := c.GetOvnEip(eipName) - if err != nil { - klog.Errorf("failed to get eip, %v", err) - return err - } // should delete if !cachedSnat.DeletionTimestamp.IsZero() { - klog.V(3).Infof("ovn delete snat %s", key) - // ovn delete snat + klog.Infof("ovn delete snat %s", key) if cachedSnat.Status.Vpc != "" && cachedSnat.Status.V4Eip != "" && cachedSnat.Status.V4IpCidr != "" { if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V4Eip, cachedSnat.Status.V4IpCidr); err != nil { klog.Errorf("failed to delete snat, %v", err) return err } } - // reset eip c.resetOvnEipQueue.Add(cachedSnat.Spec.OvnEip) return nil } - + klog.Infof("handle update ovn snat %s", key) + // check eip + eipName := cachedSnat.Spec.OvnEip + if eipName == "" { + err := fmt.Errorf("failed to create ovn snat rule, should set eip") + klog.Error(err) + return err + } + cachedEip, err := c.GetOvnEip(eipName) + if err != nil { + klog.Errorf("failed to get eip, %v", err) + return err + } if cachedEip.Spec.Type == util.Lsp { // eip is using by ecmp nexthop lsp, nat can not use err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) klog.Error(err) return err } - + if cachedEip.Status.V4Ip == "" { + err := fmt.Errorf("failed to create v4 snat %s, eip %s has no v4 ip", cachedSnat.Name, eipName) + klog.Error(err) + return err + } var v4IpCidr, vpcName string - if cachedSnat.Spec.VpcSubnet != "" { + if cachedSnat.Spec.Vpc != "" { + vpcName = cachedSnat.Spec.Vpc + } + if cachedSnat.Spec.V4IpCidr != "" { + v4IpCidr = cachedSnat.Spec.V4IpCidr + } + if v4IpCidr == "" && cachedSnat.Spec.VpcSubnet != "" { subnet, err := c.subnetsLister.Get(cachedSnat.Spec.VpcSubnet) if err != nil { klog.Errorf("failed to get vpc subnet %s, %v", cachedSnat.Spec.VpcSubnet, err) @@ -327,7 +345,7 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { vpcName = subnet.Spec.Vpc v4IpCidr = subnet.Spec.CIDRBlock } - if cachedSnat.Spec.IPName != "" { + if v4IpCidr == "" && cachedSnat.Spec.IPName != "" { vpcPodIP, err := c.ipsLister.Get(cachedSnat.Spec.IPName) if err != nil { klog.Errorf("failed to get pod ip %s, %v", cachedSnat.Spec.IPName, err) @@ -341,10 +359,21 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { vpcName = subnet.Spec.Vpc v4IpCidr = vpcPodIP.Spec.V4IPAddress } - if v4IpCidr == "" { // only support IPv4 snat err = fmt.Errorf("failed to get v4 internal ip for snat %s", key) + klog.Error(err) + return err + } + if vpcName == "" { + err := fmt.Errorf("failed to get vpc for snat %s", cachedSnat.Name) + klog.Error(err) + return err + } + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) return err } // snat change eip @@ -398,7 +427,6 @@ func (c *Controller) handleDelOvnSnatRule(key string) error { klog.Errorf("failed to remove finalizer for ovn snat %s, %v", cachedSnat.Name, err) return err } - // reset eip if cachedSnat.Spec.OvnEip != "" { c.resetOvnEipQueue.Add(cachedSnat.Spec.OvnEip) } @@ -443,12 +471,16 @@ func (c *Controller) patchOvnSnatStatus(key, vpc, v4Eip, v4IpCidr string, ready snat.Status.Ready = ready changed = true } - if (v4Eip != "" && snat.Status.V4Eip != v4Eip) || - (v4IpCidr != "" && snat.Status.V4IpCidr != v4IpCidr) || - (vpc != "" && snat.Status.Vpc != vpc) { + if vpc != "" && snat.Status.Vpc != vpc { + snat.Status.Vpc = vpc + changed = true + } + if v4Eip != "" && snat.Status.V4Eip != v4Eip { snat.Status.V4Eip = v4Eip + changed = true + } + if v4IpCidr != "" && snat.Status.V4IpCidr != v4IpCidr { snat.Status.V4IpCidr = v4IpCidr - snat.Status.Vpc = vpc changed = true } if changed { diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index bab950a0004..b383962e7ed 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -977,9 +977,16 @@ func (c *Controller) handleDeletePod(key string) error { return err } } - if exGwEnabled == "true" { - if err := c.OVNNbClient.DeleteNat(vpc.Name, "", "", address.IP); err != nil { - return err + if c.config.EnableEipSnat { + if pod.Annotations[util.EipAnnotation] != "" { + if err = c.OVNNbClient.DeleteNat(c.config.ClusterRouter, ovnnb.NATTypeDNATAndSNAT, pod.Annotations[util.EipAnnotation], address.IP); err != nil { + klog.Errorf("failed to delete nat rules: %v", err) + } + } + if pod.Annotations[util.SnatAnnotation] != "" { + if err = c.OVNNbClient.DeleteNat(c.config.ClusterRouter, ovnnb.NATTypeSNAT, "", address.IP); err != nil { + klog.Errorf("failed to delete nat rules: %v", err) + } } } } diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index ff48a79fd34..29bf01ea966 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -51,6 +51,8 @@ func (c *Controller) enqueueUpdateVpc(oldObj, newObj interface{}) { !reflect.DeepEqual(oldVpc.Spec.StaticRoutes, newVpc.Spec.StaticRoutes) || !reflect.DeepEqual(oldVpc.Spec.PolicyRoutes, newVpc.Spec.PolicyRoutes) || !reflect.DeepEqual(oldVpc.Spec.VpcPeerings, newVpc.Spec.VpcPeerings) || + !reflect.DeepEqual(oldVpc.Spec.EnableExternal, newVpc.Spec.EnableExternal) || + !reflect.DeepEqual(oldVpc.Spec.EnableBfd, newVpc.Spec.EnableBfd) || !reflect.DeepEqual(oldVpc.Annotations, newVpc.Annotations) || !reflect.DeepEqual(oldVpc.Spec.ExtraExternalSubnets, newVpc.Spec.ExtraExternalSubnets) || oldVpc.Labels[util.VpcExternalLabel] != newVpc.Labels[util.VpcExternalLabel] { diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 6dc783ad70d..d7736a54234 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -1479,12 +1479,11 @@ func getShortSharedDir(uid types.UID, volumeName string) string { } func linkExists(name string) (bool, error) { - _, err := netlink.LinkByName(name) - if err == nil { - return true, nil - } else if _, ok := err.(netlink.LinkNotFoundError); ok { - return false, nil - } else { + if _, err := netlink.LinkByName(name); err != nil { + if _, ok := err.(netlink.LinkNotFoundError); ok { + return false, nil + } return false, err } + return true, nil } diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index 1658c638ca1..89fc69affdb 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -43,19 +43,25 @@ func NewIPAM() *IPAM { func (ipam *IPAM) GetRandomAddress(podName, nicName string, mac *string, subnetName, poolName string, skippedAddrs []string, checkConflict bool) (string, string, string, error) { ipam.mutex.RLock() defer ipam.mutex.RUnlock() - + var v4, v6 string subnet, ok := ipam.Subnets[subnetName] if !ok { return "", "", "", ErrNoAvailable } v4IP, v6IP, macStr, err := subnet.GetRandomAddress(poolName, podName, nicName, mac, skippedAddrs, checkConflict) + if v4IP != nil { + v4 = v4IP.String() + } + if v6IP != nil { + v6 = v6IP.String() + } if poolName == "" { - klog.Infof("allocate v4 %s v6 %s mac %s for %s from subnet %s", v4IP, v6IP, macStr, podName, subnetName) + klog.Infof("allocate v4 %s, v6 %s, mac %s for %s from subnet %s", v4, v6, macStr, podName, subnetName) } else { - klog.Infof("allocate v4 %s v6 %s mac %s for %s from ippool %s in subnet %s", v4IP, v6IP, macStr, podName, poolName, subnetName) + klog.Infof("allocate v4 %s, v6 %s, mac %s for %s from ippool %s in subnet %s", v4, v6, macStr, podName, poolName, subnetName) } - return v4IP.String(), v6IP.String(), macStr, err + return v4, v6, macStr, err } func (ipam *IPAM) GetStaticAddress(podName, nicName, ip string, mac *string, subnetName string, checkConflict bool) (string, string, string, error) { @@ -71,7 +77,7 @@ func (ipam *IPAM) GetStaticAddress(podName, nicName, ip string, mac *string, sub var ips []IP var err error var ipAddr IP - var macStr string + var v4, v6, macStr string for _, ipStr := range strings.Split(ip, ",") { ip, err := NewIP(ipStr) if err != nil { @@ -93,14 +99,20 @@ func (ipam *IPAM) GetStaticAddress(podName, nicName, ip string, mac *string, sub switch subnet.Protocol { case kubeovnv1.ProtocolIPv4: - klog.Infof("allocate v4 %s mac %s for %s from subnet %s", ip, macStr, podName, subnetName) + klog.Infof("allocate v4 %s, mac %s for %s from subnet %s", ip, macStr, podName, subnetName) return ip, "", macStr, err case kubeovnv1.ProtocolIPv6: - klog.Infof("allocate v6 %s mac %s for %s from subnet %s", ip, macStr, podName, subnetName) + klog.Infof("allocate v6 %s, mac %s for %s from subnet %s", ip, macStr, podName, subnetName) return "", ip, macStr, err case kubeovnv1.ProtocolDual: - klog.Infof("allocate v4 %s v6 %s mac %s for %s from subnet %s", ips[0].String(), ips[1].String(), macStr, podName, subnetName) - return ips[0].String(), ips[1].String(), macStr, err + if ips[0] != nil { + v4 = ips[0].String() + } + if ips[1] != nil { + v6 = ips[1].String() + } + klog.Infof("allocate v4 %s, v6 %s, mac %s for %s from subnet %s", ips[0].String(), ips[1].String(), macStr, podName, subnetName) + return v4, v6, macStr, err } return "", "", "", ErrNoAvailable } diff --git a/pkg/ovs/ovn-nb-nat.go b/pkg/ovs/ovn-nb-nat.go index c62ce6bbc6c..967c1e6d070 100644 --- a/pkg/ovs/ovn-nb-nat.go +++ b/pkg/ovs/ovn-nb-nat.go @@ -16,6 +16,17 @@ import ( ) func (c *OVNNbClient) AddNat(lrName, natType, externalIP, logicalIP, logicalMac, port string, options map[string]string) error { + // The logical_port and external_mac are only accepted + // when router is a distributed router (rather than a gateway router) + // and type is dnat_and_snat. The logical_port is the name + // of an existing logical switch port where the logical_ip resides. + // The external_mac is an Ethernet address. + + // When the logical_port and external_mac are specified, + // the NAT rule will be programmed on the chassis where the logical_port resides. + // This includes ARP replies for the external_ip, which return the value of external_mac. + // All packets transmitted with source IP address equal to external_ip will be sent using the external_mac. + nat, err := c.newNat(lrName, natType, externalIP, logicalIP, logicalMac, port, func(nat *ovnnb.NAT) { if len(options) == 0 { return @@ -236,11 +247,35 @@ func (c *OVNNbClient) GetNATByUUID(uuid string) (*ovnnb.NAT, error) { func (c *OVNNbClient) GetNat(lrName, natType, externalIP, logicalIP string, ignoreNotFound bool) (*ovnnb.NAT, error) { // this is necessary because may exist same nat rule in different logical router if len(lrName) == 0 { - return nil, fmt.Errorf("the logical router name is required") + err := fmt.Errorf("the logical router name is required") + klog.Error(err) + return nil, err } - if natType == ovnnb.NATTypeDNAT { - return nil, fmt.Errorf("does not support dnat for now") + err := fmt.Errorf("does not support dnat for now") + klog.Error(err) + return nil, err + } + + if natType != ovnnb.NATTypeSNAT && natType != ovnnb.NATTypeDNATAndSNAT { + err := fmt.Errorf("nat type must one of [ snat, dnat_and_snat ]") + klog.Error(err) + return nil, err + } + + if natType == ovnnb.NATTypeSNAT { + if logicalIP == "" { + err := fmt.Errorf("logical ip is required when nat type is %s", natType) + klog.Error(err) + return nil, err + } + } + if natType == ovnnb.NATTypeDNATAndSNAT { + if externalIP == "" { + err := fmt.Errorf("external ip is required when nat type is %s", natType) + klog.Error(err) + return nil, err + } } fnFilter := func(nat *ovnnb.NAT) bool { @@ -263,11 +298,15 @@ func (c *OVNNbClient) GetNat(lrName, natType, externalIP, logicalIP string, igno if ignoreNotFound { return nil, nil } - return nil, fmt.Errorf("not found logical router %s nat 'type %s external ip %s logical ip %s'", lrName, natType, externalIP, logicalIP) + err := fmt.Errorf("not found logical router %s nat 'type %s external ip %s logical ip %s'", lrName, natType, externalIP, logicalIP) + klog.Error(err) + return nil, err } if len(natList) > 1 { - return nil, fmt.Errorf("more than one nat 'type %s external ip %s logical ip %s' in logical router %s", natType, externalIP, logicalIP, lrName) + err := fmt.Errorf("more than one nat 'type %s external ip %s logical ip %s' in logical router %s", natType, externalIP, logicalIP, lrName) + klog.Error(err) + return nil, err } return natList[0], nil @@ -284,17 +323,40 @@ func (c *OVNNbClient) NatExists(lrName, natType, externalIP, logicalIP string) ( } // newNat return net with basic information +// a nat rule is uniquely identified by router(lrName), type(natType) and logical_ip when snat +// a nat rule is uniquely identified by router(lrName), type(natType) and external_ip when dnat_and_snat func (c *OVNNbClient) newNat(lrName, natType, externalIP, logicalIP, logicalMac, port string, options ...func(nat *ovnnb.NAT)) (*ovnnb.NAT, error) { if len(lrName) == 0 { - return nil, fmt.Errorf("the logical router name is required") + err := fmt.Errorf("the logical router name is required") + klog.Error(err) + return nil, err + } + + if natType == ovnnb.NATTypeDNAT { + err := fmt.Errorf("does not support dnat for now") + klog.Error(err) + return nil, err } if natType != ovnnb.NATTypeSNAT && natType != ovnnb.NATTypeDNATAndSNAT { - return nil, fmt.Errorf("nat type must one of [ snat, dnat_and_snat ]") + err := fmt.Errorf("nat type must one of [ snat, dnat_and_snat ]") + klog.Error(err) + return nil, err } - if len(externalIP) == 0 || len(logicalIP) == 0 { - return nil, fmt.Errorf("nat 'externalIP %s' and 'logicalIP %s' is required", externalIP, logicalIP) + if natType == ovnnb.NATTypeSNAT { + if logicalIP == "" { + err := fmt.Errorf("logical ip is required when nat type is %s", natType) + klog.Error(err) + return nil, err + } + } + if natType == ovnnb.NATTypeDNATAndSNAT { + if externalIP == "" { + err := fmt.Errorf("external ip is required when nat type is %s", natType) + klog.Error(err) + return nil, err + } } exists, err := c.NatExists(lrName, natType, externalIP, logicalIP) diff --git a/pkg/webhook/ovn_nat_gateway.go b/pkg/webhook/ovn_nat_gateway.go index 756880d7792..4faafd45030 100644 --- a/pkg/webhook/ovn_nat_gateway.go +++ b/pkg/webhook/ovn_nat_gateway.go @@ -52,7 +52,7 @@ func (v *ValidatingHook) ovnEipUpdateHook(ctx context.Context, req admission.Req if eipOld.Spec != eipNew.Spec { if eipOld.Status.Ready { - err := fmt.Errorf("ovnEip \"%s\" is ready, not support change", eipNew.Name) + err := fmt.Errorf("OvnEip not support change") return ctrlwebhook.Errored(http.StatusBadRequest, err) } if err := v.ValidateOvnEip(ctx, &eipNew); err != nil { @@ -70,27 +70,27 @@ func (v *ValidatingHook) isOvnEipInUse(ctx context.Context, eipV4IP string) (str opts := cli.MatchingLabels{util.EipV4IpLabel: eipV4IP} err = v.cache.List(ctx, &dnatList, opts) if err != nil { - klog.Errorf("failed to get ovn dnats, %v", err) + klog.Errorf("failed to list ovn dnat, %v", err) return "", err } if len(dnatList.Items) != 0 { - return "dnat", nil + return util.DnatUsingEip, nil } err = v.cache.List(ctx, &fipList, opts) if err != nil { - klog.Errorf("failed to get ovn fip, %v", err) + klog.Errorf("failed to list ovn fip, %v", err) return "", err } if len(fipList.Items) != 0 { - return "fip", nil + return util.FipUsingEip, nil } err = v.cache.List(ctx, &snatList, opts) if err != nil { - klog.Errorf("failed to get ovn dnats, %v", err) + klog.Errorf("failed to list ovn snat, %v", err) return "", err } if len(snatList.Items) != 0 { - return "snat", nil + return util.SnatUsingEip, nil } return "", nil } @@ -105,7 +105,7 @@ func (v *ValidatingHook) ovnEipDeleteHook(ctx context.Context, req admission.Req var err error nat, err := v.isOvnEipInUse(ctx, eip.Spec.V4Ip) if nat != "" { - err = fmt.Errorf("eip \"%s\" is still using by ovn nat", eip.Name) + err = fmt.Errorf("OvnEip %s is still using by ovn nat", eip.Name) return ctrlwebhook.Errored(http.StatusBadRequest, err) } if err != nil { @@ -142,7 +142,7 @@ func (v *ValidatingHook) ovnDnatUpdateHook(ctx context.Context, req admission.Re if dnatOld.Spec != dnatNew.Spec { if dnatOld.Status.Ready { - err := fmt.Errorf("OvnDnatRule \"%s\" is ready, not support change", dnatNew.Name) + err := fmt.Errorf("OvnDnat not support change") return ctrlwebhook.Errored(http.StatusBadRequest, err) } if err := v.ValidateOvnDnat(ctx, &dnatNew); err != nil { @@ -179,7 +179,7 @@ func (v *ValidatingHook) ovnSnatUpdateHook(ctx context.Context, req admission.Re if snatOld.Spec != snatNew.Spec { if snatOld.Status.Ready { - err := fmt.Errorf("OvnSnatRule \"%s\" is ready, not support change", snatNew.Name) + err := fmt.Errorf("OvnSnat not support change") return ctrlwebhook.Errored(http.StatusBadRequest, err) } if err := v.ValidateOvnSnat(ctx, &snatNew); err != nil { @@ -216,7 +216,7 @@ func (v *ValidatingHook) ovnFipUpdateHook(ctx context.Context, req admission.Req if fipNew.Spec != fipOld.Spec { if fipOld.Status.Ready { - err := fmt.Errorf("OvnFIPRule \"%s\" is ready, not support change", fipNew.Name) + err := fmt.Errorf("OvnFip not support change") return ctrlwebhook.Errored(http.StatusBadRequest, err) } if err := v.ValidateOvnFip(ctx, &fipNew); err != nil { @@ -236,12 +236,12 @@ func (v *ValidatingHook) ValidateOvnEip(ctx context.Context, eip *ovnv1.OvnEip) if eip.Spec.V4Ip != "" { if net.ParseIP(eip.Spec.V4Ip) == nil { - err := fmt.Errorf("v4ip %s is not a valid", eip.Spec.V4Ip) + err := fmt.Errorf("spec v4Ip %s is not a valid", eip.Spec.V4Ip) return err } if !util.CIDRContainIP(subnet.Spec.CIDRBlock, eip.Spec.V4Ip) { - err := fmt.Errorf("the vip %s is not in the range of subnet \"%s\", cidr %v", + err := fmt.Errorf("the vip %s is not in the range of subnet %s, cidr %v", eip.Spec.V4Ip, subnet.Name, subnet.Spec.CIDRBlock) return err } @@ -249,12 +249,12 @@ func (v *ValidatingHook) ValidateOvnEip(ctx context.Context, eip *ovnv1.OvnEip) if eip.Spec.V6Ip != "" { if net.ParseIP(eip.Spec.V6Ip) == nil { - err := fmt.Errorf("v6ip %s is not a valid", eip.Spec.V6Ip) + err := fmt.Errorf("spec v6ip %s is not a valid", eip.Spec.V6Ip) return err } if !util.CIDRContainIP(subnet.Spec.CIDRBlock, eip.Spec.V6Ip) { - err := fmt.Errorf("the vip %s is not in the range of subnet \"%s\", cidr %v", + err := fmt.Errorf("the vip %s is not in the range of subnet %s, cidr %v", eip.Spec.V6Ip, subnet.Name, subnet.Spec.CIDRBlock) return err } @@ -265,11 +265,11 @@ func (v *ValidatingHook) ValidateOvnEip(ctx context.Context, eip *ovnv1.OvnEip) func (v *ValidatingHook) ValidateOvnDnat(ctx context.Context, dnat *ovnv1.OvnDnatRule) error { if dnat.Spec.OvnEip == "" { - err := fmt.Errorf("parameter \"OvnEip\" cannot be empty") + err := fmt.Errorf("should set spec ovnEip") return err } - if dnat.Spec.IPName == "" { - err := fmt.Errorf("parameter \"IPName\" cannot be empty") + if dnat.Spec.IPName == "" && dnat.Spec.V4Ip == "" { + err := fmt.Errorf("should set spec ipName or v4Ip") return err } eip := &ovnv1.OvnEip{} @@ -279,34 +279,34 @@ func (v *ValidatingHook) ValidateOvnDnat(ctx context.Context, dnat *ovnv1.OvnDna } if dnat.Spec.ExternalPort == "" { - err := fmt.Errorf("parameter \"externalPort\" cannot be empty") + err := fmt.Errorf("should set spec externalPort") return err } if dnat.Spec.InternalPort == "" { - err := fmt.Errorf("parameter \"internalPort\" cannot be empty") + err := fmt.Errorf("should set spec internalPort") return err } if port, err := strconv.Atoi(dnat.Spec.ExternalPort); err != nil { - errMsg := fmt.Errorf("failed to parse externalPort %s: %v", dnat.Spec.ExternalPort, err) + errMsg := fmt.Errorf("failed to parse spec externalPort %s: %v", dnat.Spec.ExternalPort, err) return errMsg } else if port < 0 || port > 65535 { - err := fmt.Errorf("externalPort %s is not a valid port", dnat.Spec.ExternalPort) + err := fmt.Errorf("spec externalPort %s is not a valid port", dnat.Spec.ExternalPort) return err } if port, err := strconv.Atoi(dnat.Spec.InternalPort); err != nil { - errMsg := fmt.Errorf("failed to parse internalIP %s: %v", dnat.Spec.InternalPort, err) + errMsg := fmt.Errorf("failed to parse spec internalIP %s: %v", dnat.Spec.InternalPort, err) return errMsg } else if port < 0 || port > 65535 { - err := fmt.Errorf("internalIP %s is not a valid port", dnat.Spec.InternalPort) + err := fmt.Errorf("spec internalIP %s is not a valid port", dnat.Spec.InternalPort) return err } if !strings.EqualFold(dnat.Spec.Protocol, "tcp") && !strings.EqualFold(dnat.Spec.Protocol, "udp") { - err := fmt.Errorf("invaild iptable protocol: %s,supported params: \"tcp\", \"udp\"", dnat.Spec.Protocol) + err := fmt.Errorf("invaild dnat protocol: %s, support tcp or udp", dnat.Spec.Protocol) return err } @@ -315,13 +315,30 @@ func (v *ValidatingHook) ValidateOvnDnat(ctx context.Context, dnat *ovnv1.OvnDna func (v *ValidatingHook) ValidateOvnSnat(ctx context.Context, snat *ovnv1.OvnSnatRule) error { if snat.Spec.OvnEip == "" { - err := fmt.Errorf("parameter \"eip\" cannot be empty") + err := fmt.Errorf("should set spec OvnEip") return err } - if snat.Spec.VpcSubnet == "" && snat.Spec.IPName == "" { - err := fmt.Errorf("should set parameter \"VpcSubnet\" or \"IPName\" at least") + + if snat.Spec.VpcSubnet != "" && snat.Spec.IPName != "" { + err := fmt.Errorf("should not set spec vpcSubnet and ipName at the same time") + return err + } + + if snat.Spec.Vpc != "" && snat.Spec.V4IpCidr == "" { + err := fmt.Errorf("should set spec v4IpCidr (subnet cidr or ip address) when spec vpc is set") return err } + + if snat.Spec.Vpc == "" && snat.Spec.V4IpCidr != "" { + err := fmt.Errorf("should set spec vpc when spec v4IpCidr is set") + return err + } + + if snat.Spec.VpcSubnet == "" && snat.Spec.IPName == "" && snat.Spec.Vpc == "" && snat.Spec.V4IpCidr == "" { + err := fmt.Errorf("should set spec vpcSubnet or ipName or vpc and v4IpCidr at least") + return err + } + eip := &ovnv1.OvnEip{} key := types.NamespacedName{Name: snat.Spec.OvnEip} return v.cache.Get(ctx, key, eip) @@ -329,11 +346,11 @@ func (v *ValidatingHook) ValidateOvnSnat(ctx context.Context, snat *ovnv1.OvnSna func (v *ValidatingHook) ValidateOvnFip(ctx context.Context, fip *ovnv1.OvnFip) error { if fip.Spec.OvnEip == "" { - err := fmt.Errorf("parameter \"OvnEip\" cannot be empty") + err := fmt.Errorf("should set spec ovnEip") return err } - if fip.Spec.IPName == "" { - err := fmt.Errorf("parameter \"IPName\" cannot be empty") + if fip.Spec.IPName == "" && fip.Spec.V4Ip == "" { + err := fmt.Errorf("should set spec ipName or v4Ip") return err } eip := &ovnv1.OvnEip{} diff --git a/test/e2e/framework/ovn-dnat.go b/test/e2e/framework/ovn-dnat.go index 35692c0a0ea..6b837fcc7a2 100644 --- a/test/e2e/framework/ovn-dnat.go +++ b/test/e2e/framework/ovn-dnat.go @@ -148,7 +148,7 @@ func (c *OvnDnatRuleClient) WaitToDisappear(name string, _, timeout time.Duratio return nil } -func MakeOvnDnatRule(name, ovnEip, ipType, ipName, internalPort, externalPort, protocol string) *apiv1.OvnDnatRule { +func MakeOvnDnatRule(name, ovnEip, ipType, ipName, vpc, v4Ip, internalPort, externalPort, protocol string) *apiv1.OvnDnatRule { dnat := &apiv1.OvnDnatRule{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -157,6 +157,8 @@ func MakeOvnDnatRule(name, ovnEip, ipType, ipName, internalPort, externalPort, p OvnEip: ovnEip, IPType: ipType, IPName: ipName, + Vpc: vpc, + V4Ip: v4Ip, InternalPort: internalPort, ExternalPort: externalPort, Protocol: protocol, diff --git a/test/e2e/framework/ovn-fip.go b/test/e2e/framework/ovn-fip.go index f21883edfc6..0b39bf5caba 100644 --- a/test/e2e/framework/ovn-fip.go +++ b/test/e2e/framework/ovn-fip.go @@ -148,7 +148,7 @@ func (c *OvnFipClient) WaitToDisappear(name string, _, timeout time.Duration) er return nil } -func MakeOvnFip(name, ovnEip, ipType, ipName string) *apiv1.OvnFip { +func MakeOvnFip(name, ovnEip, ipType, ipName, vpc, v4Ip string) *apiv1.OvnFip { fip := &apiv1.OvnFip{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -157,6 +157,8 @@ func MakeOvnFip(name, ovnEip, ipType, ipName string) *apiv1.OvnFip { OvnEip: ovnEip, IPType: ipType, IPName: ipName, + Vpc: vpc, + V4Ip: v4Ip, }, } return fip diff --git a/test/e2e/framework/ovn-snat.go b/test/e2e/framework/ovn-snat.go index 9aea1ae2ef1..b85b290cee3 100644 --- a/test/e2e/framework/ovn-snat.go +++ b/test/e2e/framework/ovn-snat.go @@ -148,7 +148,7 @@ func (c *OvnSnatRuleClient) WaitToDisappear(name string, _, timeout time.Duratio return nil } -func MakeOvnSnatRule(name, ovnEip, vpcSubnet, ipName string) *apiv1.OvnSnatRule { +func MakeOvnSnatRule(name, ovnEip, vpcSubnet, ipName, vpc, v4IpCidr string) *apiv1.OvnSnatRule { snat := &apiv1.OvnSnatRule{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -157,6 +157,8 @@ func MakeOvnSnatRule(name, ovnEip, vpcSubnet, ipName string) *apiv1.OvnSnatRule OvnEip: ovnEip, VpcSubnet: vpcSubnet, IPName: ipName, + Vpc: vpc, + V4IpCidr: v4IpCidr, }, } return snat diff --git a/test/e2e/framework/vip.go b/test/e2e/framework/vip.go index 4a7751bd55e..d036097ab43 100644 --- a/test/e2e/framework/vip.go +++ b/test/e2e/framework/vip.go @@ -123,16 +123,17 @@ func (c *VipClient) WaitToDisappear(name string, _, timeout time.Duration) error return nil } -func MakeVip(name, subnet, v4ip, v6ip, vipType string) *apiv1.Vip { +func MakeVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *apiv1.Vip { vip := &apiv1.Vip{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: apiv1.VipSpec{ - Subnet: subnet, - V4ip: v4ip, - V6ip: v6ip, - Type: vipType, + Namespace: namespaceName, + Subnet: subnet, + V4ip: v4ip, + V6ip: v6ip, + Type: vipType, }, } return vip diff --git a/test/e2e/iptables-vpc-nat-gw/e2e_test.go b/test/e2e/iptables-vpc-nat-gw/e2e_test.go index f1cc74b9857..83912a810b3 100644 --- a/test/e2e/iptables-vpc-nat-gw/e2e_test.go +++ b/test/e2e/iptables-vpc-nat-gw/e2e_test.go @@ -383,7 +383,7 @@ var _ = framework.Describe("[group:iptables-vpc-nat-gw]", func() { ) ginkgo.By("Creating iptables vip for fip") - fipVip := framework.MakeVip(fipVipName, overlaySubnetName, "", "", "") + fipVip := framework.MakeVip(f.Namespace.Name, fipVipName, overlaySubnetName, "", "", "") _ = vipClient.CreateSync(fipVip) fipVip = vipClient.Get(fipVipName) ginkgo.By("Creating iptables eip for fip") @@ -401,7 +401,7 @@ var _ = framework.Describe("[group:iptables-vpc-nat-gw]", func() { _ = iptablesSnatRuleClient.CreateSync(snat) ginkgo.By("Creating iptables vip for dnat") - dnatVip := framework.MakeVip(dnatVipName, overlaySubnetName, "", "", "") + dnatVip := framework.MakeVip(f.Namespace.Name, dnatVipName, overlaySubnetName, "", "", "") _ = vipClient.CreateSync(dnatVip) dnatVip = vipClient.Get(dnatVipName) ginkgo.By("Creating iptables eip for dnat") @@ -413,7 +413,7 @@ var _ = framework.Describe("[group:iptables-vpc-nat-gw]", func() { // share eip case ginkgo.By("Creating share vip") - shareVip := framework.MakeVip(sharedVipName, overlaySubnetName, "", "", "") + shareVip := framework.MakeVip(f.Namespace.Name, sharedVipName, overlaySubnetName, "", "", "") _ = vipClient.CreateSync(shareVip) fipVip = vipClient.Get(fipVipName) ginkgo.By("Creating share iptables eip") diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 5266297c14f..82ec6913b17 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -54,20 +54,20 @@ func makeOvnEip(name, subnet, v4ip, v6ip, mac, usage string) *kubeovnv1.OvnEip { return framework.MakeOvnEip(name, subnet, v4ip, v6ip, mac, usage) } -func makeOvnVip(name, subnet, v4ip, v6ip, vipType string) *kubeovnv1.Vip { - return framework.MakeVip(name, subnet, v4ip, v6ip, vipType) +func makeOvnVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *kubeovnv1.Vip { + return framework.MakeVip(namespaceName, name, subnet, v4ip, v6ip, vipType) } -func makeOvnFip(name, ovnEip, ipType, ipName string) *kubeovnv1.OvnFip { - return framework.MakeOvnFip(name, ovnEip, ipType, ipName) +func makeOvnFip(name, ovnEip, ipType, ipName, vpc, v4Ip string) *kubeovnv1.OvnFip { + return framework.MakeOvnFip(name, ovnEip, ipType, ipName, vpc, v4Ip) } -func makeOvnSnat(name, ovnEip, vpcSubnet, ipName string) *kubeovnv1.OvnSnatRule { - return framework.MakeOvnSnatRule(name, ovnEip, vpcSubnet, ipName) +func makeOvnSnat(name, ovnEip, vpcSubnet, ipName, vpc, v4IpCidr string) *kubeovnv1.OvnSnatRule { + return framework.MakeOvnSnatRule(name, ovnEip, vpcSubnet, ipName, vpc, v4IpCidr) } -func makeOvnDnat(name, ovnEip, ipType, ipName, internalPort, externalPort, protocol string) *kubeovnv1.OvnDnatRule { - return framework.MakeOvnDnatRule(name, ovnEip, ipType, ipName, internalPort, externalPort, protocol) +func makeOvnDnat(name, ovnEip, ipType, ipName, vpc, v4Ip, internalPort, externalPort, protocol string) *kubeovnv1.OvnDnatRule { + return framework.MakeOvnDnatRule(name, ovnEip, ipType, ipName, vpc, v4Ip, internalPort, externalPort, protocol) } var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { @@ -76,7 +76,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { var skip bool var itFn func(bool) var cs clientset.Interface - var nodeNames []string + var dockerNetwork *dockertypes.NetworkResource + var nodeNames, gwNodeNames, providerBridgeIps []string var clusterName, providerNetworkName, vlanName, underlaySubnetName, noBfdVpcName, bfdVpcName, noBfdSubnetName, bfdSubnetName string var linkMap map[string]*iproute.Link var providerNetworkClient *framework.ProviderNetworkClient @@ -84,21 +85,27 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { var vpcClient *framework.VpcClient var subnetClient *framework.SubnetClient var ovnEipClient *framework.OvnEipClient - var fipVipName, fipEipName, fipName, dnatVipName, dnatEipName, dnatName, snatEipName, snatName, namespaceName string var ipClient *framework.IPClient var vipClient *framework.VipClient var ovnFipClient *framework.OvnFipClient var ovnSnatRuleClient *framework.OvnSnatRuleClient var ovnDnatRuleClient *framework.OvnDnatRuleClient - var arpProxyVip1Name, arpProxyVip2Name string - var podClient *framework.PodClient - var dockerNetwork *dockertypes.NetworkResource + var aapVip1Name, aapVip2Name string + var lrpEipSnatName string + var dnatVipName, dnatEipName, dnatName string + var fipVipName, fipEipName, fipName string + var snatEipName, snatName string + var ipDnatVipName, ipDnatEipName, ipDnatName string + var ipFipVipName, ipFipEipName, ipFipName string + var cidrSnatEipName, cidrSnatName, ipSnatVipName, ipSnatEipName, ipSnatName string + var containerID string - var image string + var image, namespaceName string - var sharedVipName, sharedEipName, sharedEipDnatName, sharedEipSnatName, sharedEipFipShoudOkName, sharedEipFipShoudFailName string + var sharedVipName, sharedEipDnatName, sharedEipFipShoudOkName, sharedEipFipShoudFailName string + var fipPodName, podEipName, podFipName string ginkgo.BeforeEach(func() { cs = f.ClientSet @@ -116,15 +123,19 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { podClient = f.PodClient() namespaceName = f.Namespace.Name + + gwNodeNum := 2 + // gw node is 2 means e2e HA cluster will have 2 gw nodes and a worker node + // in this env, tcpdump gw nat flows will be more clear + noBfdVpcName = "no-bfd-vpc-" + framework.RandomSuffix() bfdVpcName = "bfd-vpc-" + framework.RandomSuffix() - // test arp proxy vip - // should have the same mac, which is vpc overlay subnet gw mac - arpProxyVip1Name = "arp-proxy-vip1-" + framework.RandomSuffix() - arpProxyVip2Name = "arp-proxy-vip2-" + framework.RandomSuffix() - // test allow address pair vip + aapVip1Name = "aap-vip1-" + framework.RandomSuffix() + aapVip2Name = "aap-vip2-" + framework.RandomSuffix() + + // nats use ip crd name or vip crd fipVipName = "fip-vip-" + framework.RandomSuffix() fipEipName = "fip-eip-" + framework.RandomSuffix() fipName = "fip-" + framework.RandomSuffix() @@ -136,6 +147,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { snatEipName = "snat-eip-" + framework.RandomSuffix() snatName = "snat-" + framework.RandomSuffix() noBfdSubnetName = "no-bfd-subnet-" + framework.RandomSuffix() + lrpEipSnatName = "lrp-eip-snat-" + framework.RandomSuffix() bfdSubnetName = "bfd-subnet-" + framework.RandomSuffix() providerNetworkName = "external" vlanName = "vlan-" + framework.RandomSuffix() @@ -143,12 +155,32 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // sharing case sharedVipName = "shared-vip-" + framework.RandomSuffix() - sharedEipName = "shared-eip-" + framework.RandomSuffix() sharedEipDnatName = "shared-eip-dnat-" + framework.RandomSuffix() - sharedEipSnatName = "shared-eip-snat-" + framework.RandomSuffix() sharedEipFipShoudOkName = "shared-eip-fip-should-ok-" + framework.RandomSuffix() sharedEipFipShoudFailName = "shared-eip-fip-should-fail-" + framework.RandomSuffix() + // pod with fip + fipPodName = "fip-pod-" + framework.RandomSuffix() + podEipName = fipPodName + podFipName = fipPodName + + // fip use ip addr + ipFipVipName = "ip-fip-vip-" + framework.RandomSuffix() + ipFipEipName = "ip-fip-eip-" + framework.RandomSuffix() + ipFipName = "ip-fip-" + framework.RandomSuffix() + + // dnat use ip addr + ipDnatVipName = "ip-dnat-vip-" + framework.RandomSuffix() + ipDnatEipName = "ip-dnat-eip-" + framework.RandomSuffix() + ipDnatName = "ip-dnat-" + framework.RandomSuffix() + + // snat use ip cidr + cidrSnatEipName = "cidr-snat-eip-" + framework.RandomSuffix() + cidrSnatName = "cidr-snat-" + framework.RandomSuffix() + ipSnatVipName = "ip-snat-vip-" + framework.RandomSuffix() + ipSnatEipName = "ip-snat-eip-" + framework.RandomSuffix() + ipSnatName = "ip-snat-" + framework.RandomSuffix() + containerID = "" if image == "" { image = framework.GetKubeOvnImage(cs) @@ -193,12 +225,13 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { linkMap = make(map[string]*iproute.Link, len(nodes)) nodeNames = make([]string, 0, len(nodes)) - // node ext gw ovn eip name is the same as node name in this scenario + gwNodeNames = make([]string, 0, gwNodeNum) + providerBridgeIps = make([]string, 0, len(nodes)) - for _, node := range nodes { + // node ext gw ovn eip name is the same as node name in this scenario + for index, node := range nodes { links, err := node.ListLinks() framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) - for _, link := range links { if link.Address == node.NetworkSettings.Networks[dockerNetworkName].MacAddress { linkMap[node.ID] = &link @@ -208,6 +241,9 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectHaveKey(linkMap, node.ID) linkMap[node.Name()] = linkMap[node.ID] nodeNames = append(nodeNames, node.Name()) + if index < gwNodeNum { + gwNodeNames = append(gwNodeNames, node.Name()) + } } itFn = func(exchangeLinkName bool) { @@ -257,6 +293,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { port = &links[i] } else if link.IfName == bridgeName { bridge = &links[i] + ginkgo.By("get provider bridge v4 ip " + bridge.AddrInfo[0].Local) + providerBridgeIps = append(providerBridgeIps, bridge.AddrInfo[0].Local) } if port != nil && bridge != nil { break @@ -306,9 +344,9 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Deleting ovn eip " + snatEipName) ovnEipClient.DeleteSync(snatEipName) - ginkgo.By("Deleting ovn arp proxy vip " + arpProxyVip1Name) - vipClient.DeleteSync(arpProxyVip1Name) - ginkgo.By("Deleting ovn arp proxy vip " + arpProxyVip2Name) + ginkgo.By("Deleting ovn allowed address pair vip " + aapVip1Name) + vipClient.DeleteSync(aapVip1Name) + ginkgo.By("Deleting ovn allowed address pair vip " + aapVip2Name) // clean up share eip case resource ginkgo.By("Deleting share ovn dnat " + sharedEipDnatName) @@ -317,10 +355,33 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ovnFipClient.DeleteSync(sharedEipFipShoudOkName) ginkgo.By("Deleting share ovn fip " + sharedEipFipShoudFailName) ovnFipClient.DeleteSync(sharedEipFipShoudFailName) - ginkgo.By("Deleting share ovn snat " + sharedEipSnatName) - ovnSnatRuleClient.DeleteSync(sharedEipSnatName) - - vipClient.DeleteSync(arpProxyVip2Name) + ginkgo.By("Deleting share ovn snat " + lrpEipSnatName) + ovnSnatRuleClient.DeleteSync(lrpEipSnatName) + + // clean up nats with ip or ip cidr + ginkgo.By("Deleting ovn dnat " + ipDnatName) + ovnDnatRuleClient.DeleteSync(ipDnatName) + ginkgo.By("Deleting ovn snat " + ipSnatName) + ovnSnatRuleClient.DeleteSync(ipSnatName) + ginkgo.By("Deleting ovn fip " + ipFipName) + ovnFipClient.DeleteSync(ipFipName) + + ginkgo.By("Deleting ovn eip " + ipFipEipName) + ovnFipClient.DeleteSync(ipFipEipName) + ginkgo.By("Deleting ovn eip " + ipDnatEipName) + ovnEipClient.DeleteSync(ipDnatEipName) + ginkgo.By("Deleting ovn eip " + ipSnatEipName) + ovnEipClient.DeleteSync(ipSnatEipName) + + ginkgo.By("Deleting ovn vip " + ipFipVipName) + vipClient.DeleteSync(ipFipVipName) + ginkgo.By("Deleting ovn vip " + ipDnatVipName) + vipClient.DeleteSync(ipDnatVipName) + ginkgo.By("Deleting ovn vip " + ipSnatVipName) + vipClient.DeleteSync(ipSnatVipName) + + ginkgo.By("Deleting ovn vip " + aapVip2Name) + vipClient.DeleteSync(aapVip2Name) ginkgo.By("Deleting ovn vip " + dnatVipName) vipClient.DeleteSync(dnatVipName) ginkgo.By("Deleting ovn vip " + fipVipName) @@ -328,6 +389,14 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Deleting ovn share vip " + sharedVipName) vipClient.DeleteSync(sharedVipName) + // clean fip pod + ginkgo.By("Deleting pod fip " + podFipName) + ovnFipClient.DeleteSync(podFipName) + ginkgo.By("Deleting pod with fip " + fipPodName) + podClient.DeleteSync(fipPodName) + ginkgo.By("Deleting pod eip " + podEipName) + ovnEipClient.DeleteSync(podEipName) + ginkgo.By("Deleting subnet " + noBfdSubnetName) subnetClient.DeleteSync(noBfdSubnetName) ginkgo.By("Deleting subnet " + bfdSubnetName) @@ -364,7 +433,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } }) - framework.ConformanceIt("ovn eip fip snat dnat", func() { + framework.ConformanceIt("Test ovn eip fip snat dnat", func() { ginkgo.By("Getting docker network " + dockerNetworkName) network, err := docker.NetworkInspect(dockerNetworkName) framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) @@ -411,7 +480,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectEqual(vlanSubnet.Spec.Vlan, vlanName) framework.ExpectNotEqual(vlanSubnet.Spec.CIDRBlock, "") - externalGwNodes := strings.Join(nodeNames, ",") + externalGwNodes := strings.Join(gwNodeNames, ",") ginkgo.By("Creating config map ovn-external-gw-config for centralized case") cmData := map[string]string{ "enable-external-gw": "true", @@ -430,61 +499,84 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { _, err = cs.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Create(context.Background(), configMap, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create") - ginkgo.By("1. Creating custom vpc enable external no bfd") + ginkgo.By("1. Test custom vpc nats using centralized external gw") noBfdSubnetV4Cidr := "192.168.0.0/24" noBfdSubnetV4Gw := "192.168.0.1" enableExternal := true disableBfd := false noBfdVpc := framework.MakeVpc(noBfdVpcName, "", enableExternal, disableBfd, nil) _ = vpcClient.CreateSync(noBfdVpc) - ginkgo.By("Creating overlay subnet enable ecmp") + ginkgo.By("Creating overlay subnet " + noBfdSubnetName) noBfdSubnet := framework.MakeSubnet(noBfdSubnetName, "", noBfdSubnetV4Cidr, noBfdSubnetV4Gw, noBfdVpcName, util.OvnProvider, nil, nil, nil) _ = subnetClient.CreateSync(noBfdSubnet) + ginkgo.By("Creating pod on nodes") + for _, node := range nodeNames { + // create pod on gw node and worker node + podOnNodeName := fmt.Sprintf("no-bfd-%s", node) + ginkgo.By("Creating no bfd pod " + podOnNodeName + " with subnet " + noBfdSubnetName) + annotations := map[string]string{util.LogicalSwitchAnnotation: noBfdSubnetName} + cmd := []string{"sh", "-c", "sleep infinity"} + pod := framework.MakePod(namespaceName, podOnNodeName, nil, annotations, image, cmd, nil) + pod.Spec.NodeName = node + _ = podClient.CreateSync(pod) + } + + ginkgo.By("Creating pod with fip") + annotations := map[string]string{util.LogicalSwitchAnnotation: noBfdSubnetName} + cmd := []string{"sh", "-c", "sleep infinity"} + fipPod := framework.MakePod(namespaceName, fipPodName, nil, annotations, image, cmd, nil) + fipPod = podClient.CreateSync(fipPod) + podEip := framework.MakeOvnEip(podEipName, underlaySubnetName, "", "", "", "") + _ = ovnEipClient.CreateSync(podEip) + fipPodIP := ovs.PodNameToPortName(fipPod.Name, fipPod.Namespace, noBfdSubnet.Spec.Provider) + podFip := framework.MakeOvnFip(podFipName, podEipName, "", fipPodIP, "", "") + podFip = ovnFipClient.CreateSync(podFip) + + ginkgo.By("1.1 Test fip dnat snat share eip by by setting eip name and ip name") + ginkgo.By("Create snat, dnat, fip with the same vpc lrp eip") + noBfdlrpEipName := fmt.Sprintf("%s-%s", noBfdVpcName, underlaySubnetName) + noBfdLrpEip := ovnEipClient.Get(noBfdlrpEipName) + lrpEipSnat := framework.MakeOvnSnatRule(lrpEipSnatName, noBfdlrpEipName, noBfdSubnetName, "", "", "") + _ = ovnSnatRuleClient.CreateSync(lrpEipSnat) + ginkgo.By("Get lrp eip snat") + lrpEipSnat = ovnSnatRuleClient.Get(lrpEipSnatName) + ginkgo.By("Check share snat should has the external ip label") + framework.ExpectHaveKeyWithValue(lrpEipSnat.Labels, util.EipV4IpLabel, noBfdLrpEip.Spec.V4Ip) - // share eip case ginkgo.By("Creating share vip") - shareVip := framework.MakeVip(sharedVipName, noBfdSubnetName, "", "", "") + shareVip := framework.MakeVip(namespaceName, sharedVipName, noBfdSubnetName, "", "", "") _ = vipClient.CreateSync(shareVip) - ginkgo.By("Creating share ovn eip") - shareEip := framework.MakeOvnEip(sharedEipName, underlaySubnetName, "", "", "", "") - _ = ovnEipClient.CreateSync(shareEip) ginkgo.By("Creating the first ovn fip with share eip vip should be ok") - shareFipShouldOk := framework.MakeOvnFip(sharedEipFipShoudOkName, sharedEipName, util.Vip, sharedVipName) + shareFipShouldOk := framework.MakeOvnFip(sharedEipFipShoudOkName, noBfdlrpEipName, util.Vip, sharedVipName, "", "") _ = ovnFipClient.CreateSync(shareFipShouldOk) ginkgo.By("Creating the second ovn fip with share eip vip should be failed") - shareFipShouldFail := framework.MakeOvnFip(sharedEipFipShoudFailName, sharedEipName, util.Vip, sharedVipName) + shareFipShouldFail := framework.MakeOvnFip(sharedEipFipShoudFailName, noBfdlrpEipName, util.Vip, sharedVipName, "", "") _ = ovnFipClient.Create(shareFipShouldFail) ginkgo.By("Creating ovn dnat for dnat with share eip vip") - shareDnat := framework.MakeOvnDnatRule(sharedEipDnatName, sharedEipName, util.Vip, sharedVipName, "80", "8080", "tcp") + shareDnat := framework.MakeOvnDnatRule(sharedEipDnatName, noBfdlrpEipName, util.Vip, sharedVipName, "", "", "80", "8080", "tcp") _ = ovnDnatRuleClient.CreateSync(shareDnat) - ginkgo.By("Creating ovn snat with share eip vip") - shareSnat := framework.MakeOvnSnatRule(sharedEipSnatName, sharedEipName, noBfdSubnetName, "") - _ = ovnSnatRuleClient.CreateSync(shareSnat) - ginkgo.By("Get share eip") - shareEip = ovnEipClient.Get(sharedEipName) + ginkgo.By("Get shared lrp eip") + noBfdLrpEip = ovnEipClient.Get(noBfdlrpEipName) ginkgo.By("Get share dnat") shareDnat = ovnDnatRuleClient.Get(sharedEipDnatName) - ginkgo.By("Get share snat") - shareSnat = ovnSnatRuleClient.Get(sharedEipSnatName) + ginkgo.By("Get share fip should ok") shareFipShouldOk = ovnFipClient.Get(sharedEipFipShoudOkName) ginkgo.By("Get share fip should fail") shareFipShouldFail = ovnFipClient.Get(sharedEipFipShoudFailName) // check ginkgo.By("Check share eip should has the external ip label") - framework.ExpectHaveKeyWithValue(shareEip.Labels, util.EipV4IpLabel, shareEip.Spec.V4Ip) + framework.ExpectHaveKeyWithValue(noBfdLrpEip.Labels, util.EipV4IpLabel, noBfdLrpEip.Spec.V4Ip) ginkgo.By("Check share dnat should has the external ip label") - framework.ExpectHaveKeyWithValue(shareDnat.Labels, util.EipV4IpLabel, shareEip.Spec.V4Ip) - ginkgo.By("Check share snat should has the external ip label") - framework.ExpectHaveKeyWithValue(shareSnat.Labels, util.EipV4IpLabel, shareEip.Spec.V4Ip) + framework.ExpectHaveKeyWithValue(shareDnat.Labels, util.EipV4IpLabel, noBfdLrpEip.Spec.V4Ip) ginkgo.By("Check share fip should ok should has the external ip label") - framework.ExpectHaveKeyWithValue(shareFipShouldOk.Labels, util.EipV4IpLabel, shareEip.Spec.V4Ip) + framework.ExpectHaveKeyWithValue(shareFipShouldOk.Labels, util.EipV4IpLabel, noBfdLrpEip.Spec.V4Ip) ginkgo.By("Check share fip should fail should not be ready") framework.ExpectEqual(shareFipShouldFail.Status.Ready, false) // make sure eip is shared nats := []string{util.DnatUsingEip, util.FipUsingEip, util.SnatUsingEip} - framework.ExpectEqual(shareEip.Status.Nat, strings.Join(nats, ",")) + framework.ExpectEqual(noBfdLrpEip.Status.Nat, strings.Join(nats, ",")) // make sure vpc has normal external static routes noBfdVpc = vpcClient.Get(noBfdVpcName) for _, route := range noBfdVpc.Spec.StaticRoutes { @@ -493,12 +585,53 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectContainSubstring(vlanSubnetGw, route.NextHopIP) } - ginkgo.By("2. Creating custom vpc enable external and bfd") + ginkgo.By("1.2 Test snat, fip external connectivity") + for _, node := range nodeNames { + // all the pods should ping lrp, node br-external ip successfully + podOnNodeName := fmt.Sprintf("no-bfd-%s", node) + pod := podClient.GetPod(podOnNodeName) + ginkgo.By("Test pod ping lrp eip " + noBfdlrpEipName) + command := fmt.Sprintf("ping -W 1 -c 1 %s", noBfdLrpEip.Status.V4Ip) + stdOutput, errOutput, err := framework.ExecShellInPod(context.Background(), f, pod.Namespace, pod.Name, command) + framework.Logf("output from exec on client pod %s dest lrp ip %s\n", pod.Name, noBfdLrpEip.Name) + if stdOutput != "" && err == nil { + framework.Logf("output:\n%s", stdOutput) + } + framework.Logf("exec %s failed err: %v, errOutput: %s, stdOutput: %s", command, err, errOutput, stdOutput) + + ginkgo.By("Test pod ping pod fip " + podFip.Status.V4Ip) + command = fmt.Sprintf("ping -W 1 -c 1 %s", podFip.Status.V4Ip) + stdOutput, errOutput, err = framework.ExecShellInPod(context.Background(), f, pod.Namespace, pod.Name, command) + framework.Logf("output from exec on client pod %s dst fip %s\n", pod.Name, noBfdLrpEip.Name) + if stdOutput != "" && err == nil { + framework.Logf("output:\n%s", stdOutput) + } + framework.Logf("exec %s failed err: %v, errOutput: %s, stdOutput: %s", command, err, errOutput, stdOutput) + + ginkgo.By("Test pod ping node provider bridge ip " + strings.Join(providerBridgeIps, ",")) + for _, ip := range providerBridgeIps { + command := fmt.Sprintf("ping -W 1 -c 1 %s", ip) + stdOutput, errOutput, err = framework.ExecShellInPod(context.Background(), f, pod.Namespace, pod.Name, command) + framework.Logf("output from exec on client pod %s dest node ip %s\n", pod.Name, ip) + if stdOutput != "" && err == nil { + framework.Logf("output:\n%s", stdOutput) + } + } + framework.Logf("exec %s failed err: %v, errOutput: %s, stdOutput: %s", command, err, errOutput, stdOutput) + } + + // nat with ip crd name and share the same external eip tests all passed + ginkgo.By("2. Test custom vpc with bfd route") + ginkgo.By("2.1 Test custom vpc dnat, fip, snat in traditonal way") + ginkgo.By("Create dnat, fip, snat with eip name and ip or ip cidr") + for _, nodeName := range nodeNames { ginkgo.By("Creating ovn node-ext-gw type eip on node " + nodeName) eip := makeOvnEip(nodeName, underlaySubnetName, "", "", "", util.Lsp) _ = ovnEipClient.CreateSync(eip) } + + // TODO:// ipv6, dual stack support bfdSubnetV4Cidr := "192.168.1.0/24" bfdSubnetV4Gw := "192.168.1.1" enableBfd := true @@ -509,48 +642,65 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { bfdSubnet.Spec.EnableEcmp = true _ = subnetClient.CreateSync(bfdSubnet) - // arp proxy vip test case - ginkgo.By("Creating two arp proxy vips, should have the same mac which is from gw subnet mac") - ginkgo.By("Creating arp proxy vip " + arpProxyVip1Name) - arpProxyVip1 := makeOvnVip(arpProxyVip1Name, bfdSubnetName, "", "", util.SwitchLBRuleVip) - _ = vipClient.CreateSync(arpProxyVip1) - ginkgo.By("Creating arp proxy vip " + arpProxyVip2Name) - arpProxyVip2 := makeOvnVip(arpProxyVip2Name, bfdSubnetName, "", "", util.SwitchLBRuleVip) - _ = vipClient.CreateSync(arpProxyVip2) - - arpProxyVip1 = vipClient.Get(arpProxyVip1Name) - arpProxyVip2 = vipClient.Get(arpProxyVip2Name) - framework.ExpectEqual(arpProxyVip1.Status.Mac, arpProxyVip2.Status.Mac) - - // allowed address pair vip test case - ginkgo.By("Creating crd in centralized case") - // for now, vip do not have parent ip can be used in centralized external gw case - ginkgo.By("Creating ovn vip " + fipVipName) - fipVip := makeOvnVip(fipVipName, bfdSubnetName, "", "", "") - _ = vipClient.CreateSync(fipVip) - ginkgo.By("Creating ovn eip " + fipEipName) - eip := makeOvnEip(fipEipName, underlaySubnetName, "", "", "", "") - _ = ovnEipClient.CreateSync(eip) - ginkgo.By("Creating ovn fip " + fipName) - fip := makeOvnFip(fipName, fipEipName, util.Vip, fipVipName) - _ = ovnFipClient.CreateSync(fip) - - ginkgo.By("Creating ovn eip " + snatEipName) - snatEip := makeOvnEip(snatEipName, underlaySubnetName, "", "", "", "") - _ = ovnEipClient.CreateSync(snatEip) - ginkgo.By("Creating ovn snat" + snatName) - snat := makeOvnSnat(snatName, snatEipName, bfdSubnetName, "") - _ = ovnSnatRuleClient.CreateSync(snat) - - ginkgo.By("Creating ovn vip " + dnatVipName) - dnatVip := makeOvnVip(dnatVipName, bfdSubnetName, "", "", "") - _ = vipClient.CreateSync(dnatVip) - ginkgo.By("Creating ovn eip " + dnatEipName) - dnatEip := makeOvnEip(dnatEipName, underlaySubnetName, "", "", "", "") - _ = ovnEipClient.CreateSync(dnatEip) - ginkgo.By("Creating ovn dnat " + dnatName) - dnat := makeOvnDnat(dnatName, dnatEipName, util.Vip, dnatVipName, "80", "8080", "tcp") - _ = ovnDnatRuleClient.CreateSync(dnat) + // TODO:// support vip type allowed address pair while use security group + + ginkgo.By("Test ovn fip with eip name and ip") + ginkgo.By("Creating ovn vip " + ipFipVipName) + ipFipVip := makeOvnVip(namespaceName, ipFipVipName, bfdSubnetName, "", "", "") + ipFipVip = vipClient.CreateSync(ipFipVip) + framework.ExpectNotEmpty(ipFipVip.Status.V4ip) + ginkgo.By("Creating ovn eip " + ipFipEipName) + ipFipEip := makeOvnEip(ipFipEipName, underlaySubnetName, "", "", "", "") + ipFipEip = ovnEipClient.CreateSync(ipFipEip) + framework.ExpectNotEmpty(ipFipEip.Status.V4Ip) + ginkgo.By("Creating ovn fip " + ipFipName) + ipFip := makeOvnFip(fipName, ipFipEipName, "", "", bfdVpcName, ipFipVip.Status.V4ip) + ipFip = ovnFipClient.CreateSync(ipFip) + framework.ExpectEqual(ipFip.Status.V4Eip, ipFipEip.Status.V4Ip) + framework.ExpectNotEmpty(ipFip.Status.V4Ip) + + ginkgo.By("Test ovn dnat with eip name and ip") + ginkgo.By("Creating ovn vip " + ipDnatVipName) + ipDnatVip := makeOvnVip(namespaceName, ipDnatVipName, bfdSubnetName, "", "", "") + ipDnatVip = vipClient.CreateSync(ipDnatVip) + framework.ExpectNotEmpty(ipDnatVip.Status.V4ip) + ginkgo.By("Creating ovn eip " + ipDnatEipName) + ipDnatEip := makeOvnEip(ipDnatEipName, underlaySubnetName, "", "", "", "") + ipDnatEip = ovnEipClient.CreateSync(ipDnatEip) + framework.ExpectNotEmpty(ipDnatEip.Status.V4Ip) + ginkgo.By("Creating ovn dnat " + ipDnatName) + ipDnat := makeOvnDnat(ipDnatName, ipDnatEipName, "", "", bfdVpcName, ipDnatVip.Status.V4ip, "80", "8080", "tcp") + ipDnat = ovnDnatRuleClient.CreateSync(ipDnat) + framework.ExpectEqual(ipDnat.Status.Vpc, bfdVpcName) + framework.ExpectEqual(ipDnat.Status.V4Eip, ipDnatEip.Status.V4Ip) + framework.ExpectEqual(ipDnat.Status.V4Ip, ipDnatVip.Status.V4ip) + + ginkgo.By("Test ovn snat with eip name and ip cidr") + ginkgo.By("Creating ovn eip " + cidrSnatEipName) + cidrSnatEip := makeOvnEip(cidrSnatEipName, underlaySubnetName, "", "", "", "") + cidrSnatEip = ovnEipClient.CreateSync(cidrSnatEip) + framework.ExpectNotEmpty(cidrSnatEip.Status.V4Ip) + ginkgo.By("Creating ovn snat mapping with subnet cidr" + bfdSubnetV4Cidr) + cidrSnat := makeOvnSnat(cidrSnatName, cidrSnatEipName, "", "", bfdVpcName, bfdSubnetV4Cidr) + cidrSnat = ovnSnatRuleClient.CreateSync(cidrSnat) + framework.ExpectEqual(cidrSnat.Status.Vpc, bfdVpcName) + framework.ExpectEqual(cidrSnat.Status.V4Eip, cidrSnatEip.Status.V4Ip) + framework.ExpectEqual(cidrSnat.Status.V4IpCidr, bfdSubnetV4Cidr) + + ginkgo.By("Test ovn snat with eip name and ip") + ginkgo.By("Creating ovn vip " + ipSnatVipName) + ipSnatVip := makeOvnVip(namespaceName, ipSnatVipName, bfdSubnetName, "", "", "") + ipSnatVip = vipClient.CreateSync(ipSnatVip) + framework.ExpectNotEmpty(ipSnatVip.Status.V4ip) + ginkgo.By("Creating ovn eip " + ipSnatEipName) + ipSnatEip := makeOvnEip(ipSnatEipName, underlaySubnetName, "", "", "", "") + ipSnatEip = ovnEipClient.CreateSync(ipSnatEip) + framework.ExpectNotEmpty(ipSnatEip.Status.V4Ip) + ginkgo.By("Creating ovn snat " + ipSnatName) + ipSnat := makeOvnSnat(ipSnatName, ipSnatEipName, "", "", bfdVpcName, ipSnatVip.Status.V4ip) + ipSnat = ovnSnatRuleClient.CreateSync(ipSnat) + framework.ExpectEqual(ipSnat.Status.Vpc, bfdVpcName) + framework.ExpectEqual(ipSnat.Status.V4IpCidr, ipSnatVip.Status.V4ip) k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) framework.ExpectNoError(err) @@ -566,14 +716,17 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectEqual(route.Policy, kubeovnv1.PolicySrc) framework.ExpectNotEmpty(route.CIDR) } - k8sNodes, err = e2enode.GetReadySchedulableNodes(context.Background(), cs) - framework.ExpectNoError(err) - for _, node := range k8sNodes.Items { - ginkgo.By("Deleting ovn eip " + node.Name) - ovnEipClient.DeleteSync(node.Name) - } - ginkgo.By("2. Updating config map ovn-external-gw-config for distributed case") + for _, node := range nodeNames { + podOnNodeName := fmt.Sprintf("bfd-%s", node) + ginkgo.By("Creating bfd pod " + podOnNodeName + " with subnet " + bfdSubnetName) + annotations := map[string]string{util.LogicalSwitchAnnotation: bfdSubnetName} + cmd := []string{"sh", "-c", "sleep infinity"} + pod := framework.MakePod(namespaceName, podOnNodeName, nil, annotations, image, cmd, nil) + pod.Spec.NodeName = node + _ = podClient.CreateSync(pod) + } + ginkgo.By("3. Updating config map ovn-external-gw-config for distributed case") cmData = map[string]string{ "enable-external-gw": "true", "external-gw-nodes": externalGwNodes, @@ -596,32 +749,38 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { nodes, err := kind.ListNodes(clusterName, "") framework.ExpectNoError(err, "getting nodes in kind cluster") framework.ExpectNotEmpty(nodes) - ginkgo.By("Creating crd in distributed case") + ginkgo.By("4. Creating crd in distributed case") for _, node := range nodeNames { - podOnNodeName := fmt.Sprintf("on-node-%s", node) + podOnNodeName := fmt.Sprintf("bfd-%s", node) eipOnNodeName := fmt.Sprintf("eip-on-node-%s", node) fipOnNodeName := fmt.Sprintf("fip-on-node-%s", node) - ginkgo.By("Creating pod " + podOnNodeName + " with subnet " + bfdSubnetName) - annotations := map[string]string{util.LogicalSwitchAnnotation: bfdSubnetName} - cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podOnNodeName, nil, annotations, image, cmd, nil) - pod.Spec.NodeName = node - _ = podClient.CreateSync(pod) - // create fip in distributed case - // for now, vip has no lsp, so not support in distributed case ipName := ovs.PodNameToPortName(podOnNodeName, namespaceName, bfdSubnet.Spec.Provider) ginkgo.By("Get pod ip" + ipName) ip := ipClient.Get(ipName) ginkgo.By("Creating ovn eip " + eipOnNodeName) - eip = makeOvnEip(eipOnNodeName, underlaySubnetName, "", "", "", "") - _ = ovnEipClient.CreateSync(eip) + eipOnNode := makeOvnEip(eipOnNodeName, underlaySubnetName, "", "", "", "") + _ = ovnEipClient.CreateSync(eipOnNode) ginkgo.By("Creating ovn fip " + fipOnNodeName) - fip := makeOvnFip(fipOnNodeName, eipOnNodeName, "", ip.Name) + fip := makeOvnFip(fipOnNodeName, eipOnNodeName, "", ip.Name, "", "") _ = ovnFipClient.CreateSync(fip) } - ginkgo.By("Deleting crd in distributed case") + // wait here to have an insight into all the ovn nat resources + ginkgo.By("4. Deleting pod") + for _, node := range nodeNames { + podOnNodeName := fmt.Sprintf("bfd-%s", node) + ginkgo.By("Deleting pod " + podOnNodeName) + podClient.DeleteSync(podOnNodeName) + podOnNodeName = fmt.Sprintf("no-bfd-%s", node) + ginkgo.By("Deleting pod " + podOnNodeName) + podClient.DeleteSync(podOnNodeName) + + } + + ginkgo.By("5. Deleting crd in distributed case") for _, node := range nodeNames { + ginkgo.By("Deleting node external gw ovn eip " + node) + ovnEipClient.DeleteSync(node) podOnNodeName := fmt.Sprintf("on-node-%s", node) eipOnNodeName := fmt.Sprintf("eip-on-node-%s", node) fipOnNodeName := fmt.Sprintf("fip-on-node-%s", node) diff --git a/test/e2e/vip/e2e_test.go b/test/e2e/vip/e2e_test.go new file mode 100644 index 00000000000..c01a07d8f28 --- /dev/null +++ b/test/e2e/vip/e2e_test.go @@ -0,0 +1,128 @@ +package vip + +import ( + "flag" + "testing" + + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + + "github.com/onsi/ginkgo/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" +) + +func makeOvnVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *kubeovnv1.Vip { + return framework.MakeVip(namespaceName, name, subnet, v4ip, v6ip, vipType) +} + +var _ = framework.Describe("[group:vip]", func() { + f := framework.NewDefaultFramework("vip") + + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var vipClient *framework.VipClient + var vpc *kubeovnv1.Vpc + var subnet *kubeovnv1.Subnet + var namespaceName, vpcName, subnetName, cidr string + + // test switch lb vip, which ip is in the vpc subnet cidr + // switch lb vip use gw mac to trigger lb nat flows + var switchLbVip1Name, switchLbVip2Name string + + // test allowed address pair vip + var vip1Name, vip2Name string + + ginkgo.BeforeEach(func() { + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + vipClient = f.VipClient() + namespaceName = f.Namespace.Name + cidr = framework.RandomCIDR(f.ClusterIPFamily) + + // should have the same mac, which mac is the same as its vpc overlay subnet gw mac + randomSuffix := framework.RandomSuffix() + switchLbVip1Name = "switch-lb-vip1-" + randomSuffix + switchLbVip2Name = "switch-lb-vip2-" + randomSuffix + + // should have different mac + vip1Name = "vip1-" + randomSuffix + vip2Name = "vip2-" + randomSuffix + + vpcName = "vpc-" + randomSuffix + subnetName = "subnet-" + randomSuffix + ginkgo.By("Creating vpc " + vpcName) + vpc = framework.MakeVpc(vpcName, "", false, false, []string{namespaceName}) + vpc = vpcClient.CreateSync(vpc) + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", vpcName, "", nil, nil, []string{namespaceName}) + subnet = subnetClient.CreateSync(subnet) + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Deleting switch lb vip " + switchLbVip1Name) + vipClient.DeleteSync(switchLbVip1Name) + ginkgo.By("Deleting switch lb vip " + switchLbVip2Name) + vipClient.DeleteSync(switchLbVip2Name) + ginkgo.By("Deleting allowed address pair vip " + vip1Name) + vipClient.DeleteSync(vip1Name) + ginkgo.By("Deleting allowed address pair vip " + vip2Name) + vipClient.DeleteSync(vip2Name) + + ginkgo.By("Deleting subnet " + subnetName) + subnetClient.DeleteSync(subnetName) + ginkgo.By("Deleting vpc " + vpcName) + vpcClient.DeleteSync(vpcName) + }) + + framework.ConformanceIt("Test vip", func() { + ginkgo.By("1. Test allowed address pair vip") + ginkgo.By("Creating allowed address pair vip, should have different ip and mac") + ginkgo.By("Creating allowed address pair vip " + vip1Name) + vip1 := makeOvnVip(namespaceName, vip1Name, subnetName, "", "", "") + vip1 = vipClient.CreateSync(vip1) + ginkgo.By("Creating allowed address pair vip " + vip2Name) + vip2 := makeOvnVip(namespaceName, vip2Name, subnetName, "", "", "") + vip2 = vipClient.CreateSync(vip2) + // arp proxy vip only used in switch lb rule, the lb vip use the subnet gw mac to use lb nat flow + framework.ExpectNotEqual(vip1.Status.Mac, vip2.Status.Mac) + if vip1.Status.V4ip != "" { + framework.ExpectNotEqual(vip1.Status.V4ip, vip2.Status.V4ip) + } else { + framework.ExpectNotEqual(vip1.Status.V6ip, vip2.Status.V6ip) + } + + ginkgo.By("2. Test switch lb vip") + ginkgo.By("Creating two arp proxy vips, should have the same mac which is from gw subnet mac") + ginkgo.By("Creating arp proxy switch lb vip " + switchLbVip1Name) + switchLbVip1 := makeOvnVip(namespaceName, switchLbVip1Name, subnetName, "", "", util.SwitchLBRuleVip) + switchLbVip1 = vipClient.CreateSync(switchLbVip1) + ginkgo.By("Creating arp proxy switch lb vip " + switchLbVip2Name) + switchLbVip2 := makeOvnVip(namespaceName, switchLbVip2Name, subnetName, "", "", util.SwitchLBRuleVip) + switchLbVip2 = vipClient.CreateSync(switchLbVip2) + // arp proxy vip only used in switch lb rule, the lb vip use the subnet gw mac to use lb nat flow + framework.ExpectEqual(switchLbVip1.Status.Mac, switchLbVip2.Status.Mac) + if vip1.Status.V4ip != "" { + framework.ExpectNotEqual(vip1.Status.V4ip, vip2.Status.V4ip) + } else { + framework.ExpectNotEqual(vip1.Status.V6ip, vip2.Status.V6ip) + } + }) +}) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + e2e.RunE2ETests(t) +} diff --git a/test/e2e/webhook/vip/vip.go b/test/e2e/webhook/vip/vip.go index a93e3e74b95..d8c89270cc0 100644 --- a/test/e2e/webhook/vip/vip.go +++ b/test/e2e/webhook/vip/vip.go @@ -53,7 +53,7 @@ var _ = framework.Describe("[group:webhook-vip]", func() { framework.ConformanceIt("check create vip with different errors", func() { ginkgo.By("Creating vip " + vipName) - vip = framework.MakeVip(vipName, "", "", "", "") + vip = framework.MakeVip(namespaceName, vipName, "", "", "", "") ginkgo.By("validating subnet") vip.Spec.Subnet = "" diff --git a/yamls/crd.yaml b/yamls/crd.yaml index 8d331320810..979ef73c3aa 100644 --- a/yamls/crd.yaml +++ b/yamls/crd.yaml @@ -1293,8 +1293,6 @@ spec: type: string v4Ip: type: string - macAddress: - type: string vpc: type: string conditions: @@ -1323,6 +1321,10 @@ spec: type: string ipName: type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1351,8 +1353,8 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string - - jsonPath: .status.v4ipCidr - name: V4Ip + - jsonPath: .status.v4IpCidr + name: V4IpCidr type: string - jsonPath: .status.ready name: Ready @@ -1368,7 +1370,7 @@ spec: type: boolean v4Eip: type: string - v4ipCidr: + v4IpCidr: type: string vpc: type: string @@ -1398,6 +1400,10 @@ spec: type: string ipName: type: string + vpc: + type: string + v4IpCidr: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1420,6 +1426,9 @@ spec: subresources: status: {} additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string - jsonPath: .spec.ovnEip name: Eip type: string @@ -1457,8 +1466,6 @@ spec: type: string v4Ip: type: string - macAddress: - type: string vpc: type: string externalPort: @@ -1501,6 +1508,10 @@ spec: type: string protocol: type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition