diff --git a/Makefile b/Makefile index ec0a743fad5..49f6d5bed35 100644 --- a/Makefile +++ b/Makefile @@ -348,18 +348,21 @@ kind-init-ipv4: kind-clean kind-init-ovn-ic: kind-init-ovn-ic-ipv4 .PHONY: kind-init-ovn-ic-ipv4 -kind-init-ovn-ic-ipv4: kind-clean-ovn-ic kind-init - @$(MAKE) kind-generate-config +kind-init-ovn-ic-ipv4: kind-clean-ovn-ic + @ovn_ic=true $(MAKE) kind-init + @ovn_ic=true $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) .PHONY: kind-init-ovn-ic-ipv6 -kind-init-ovn-ic-ipv6: kind-clean-ovn-ic kind-init-ipv6 - @ip_family=ipv6 $(MAKE) kind-generate-config +kind-init-ovn-ic-ipv6: kind-clean-ovn-ic + @ovn_ic=true $(MAKE) kind-init-ipv6 + @ovn_ic=true ip_family=ipv6 $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) .PHONY: kind-init-ovn-ic-dual -kind-init-ovn-ic-dual: kind-clean-ovn-ic kind-init-dual - @ip_family=dual $(MAKE) kind-generate-config +kind-init-ovn-ic-dual: kind-clean-ovn-ic + @ovn_ic=true $(MAKE) kind-init-dual + @ovn_ic=true ip_family=dual $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) .PHONY: kind-init-ovn-submariner @@ -511,6 +514,7 @@ kind-install-ovn-ic: kind-install-ovn-ic-ipv4 kind-install-ovn-ic-ipv4: kind-install $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) kubectl config use-context kind-kube-ovn1 + $(MAKE) kind-untaint-control-plane sed -e 's/10.16.0/10.18.0/g' \ -e 's/10.96.0/10.98.0/g' \ -e 's/100.64.0/100.68.0/g' \ @@ -521,8 +525,8 @@ kind-install-ovn-ic-ipv4: kind-install docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh @set -e; \ ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 @@ -546,8 +550,8 @@ kind-install-ovn-ic-ipv6: kind-install-ipv6 docker run -d --name ovn-ic-db --network kind -e PROTOCOL="ipv6" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh @set -e; \ ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 @@ -575,8 +579,8 @@ kind-install-ovn-ic-dual: kind-install-dual @set -e; \ ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 diff --git a/dist/images/start-ic-db.sh b/dist/images/start-ic-db.sh index fe46e843a88..3927b3b171d 100644 --- a/dist/images/start-ic-db.sh +++ b/dist/images/start-ic-db.sh @@ -1,16 +1,6 @@ #!/bin/bash set -eo pipefail -TS_NAME=${TS_NAME:-ts} -PROTOCOL=${PROTOCOL:-ipv4} -if [ "$PROTOCOL" = "ipv4" ]; then - TS_CIDR=${TS_CIDR:-169.254.100.0/24} -elif [ "$PROTOCOL" = "ipv6" ]; then - TS_CIDR=${TS_CIDR:-fe80:a9fe:64::/112} -elif [ "$PROTOCOL" = "dual" ]; then - TS_CIDR=${TS_CIDR:-"169.254.100.0/24,fe80:a9fe:64::/112"} -fi - function quit { /usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb exit 0 @@ -26,8 +16,6 @@ trap quit EXIT if [[ -z "$NODE_IPS" && -z "$LOCAL_IP" ]]; then /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes --db-ic-nb-addr="[::]" --db-ic-sb-addr="[::]" start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - ovn-ic-nbctl --may-exist ts-add "$TS_NAME" - ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR" tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else if [[ -z "$LEADER_IP" ]]; then @@ -40,8 +28,6 @@ else --ovn-ic-sb-db="$(gen_conn_str 6648)" \ start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - ovn-ic-nbctl --may-exist ts-add "$TS_NAME" - ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR" tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else echo "follower start with local ${LOCAL_IP}, leader ${LEADER_IP} and cluster $(gen_conn_str 6647)" diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index 5e326ef3627..6b3cddf4694 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -704,6 +704,20 @@ func (mr *MockLogicalSwitchPortMockRecorder) DeleteLogicalSwitchPort(lspName int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalSwitchPort", reflect.TypeOf((*MockLogicalSwitchPort)(nil).DeleteLogicalSwitchPort), lspName) } +// DeleteLogicalSwitchPorts mocks base method. +func (m *MockLogicalSwitchPort) DeleteLogicalSwitchPorts(externalIDs map[string]string, filter func(*ovnnb.LogicalSwitchPort) bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalSwitchPorts", externalIDs, filter) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalSwitchPorts indicates an expected call of DeleteLogicalSwitchPorts. +func (mr *MockLogicalSwitchPortMockRecorder) DeleteLogicalSwitchPorts(externalIDs, filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalSwitchPorts", reflect.TypeOf((*MockLogicalSwitchPort)(nil).DeleteLogicalSwitchPorts), externalIDs, filter) +} + // EnablePortLayer2forward mocks base method. func (m *MockLogicalSwitchPort) EnablePortLayer2forward(lspName string) error { m.ctrl.T.Helper() @@ -1897,6 +1911,21 @@ func (mr *MockLogicalRouterPolicyMockRecorder) DeleteLogicalRouterPolicyByUUID(l return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterPolicyByUUID", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).DeleteLogicalRouterPolicyByUUID), lrName, uuid) } +// GetLogicalRouterPoliciesByExtID mocks base method. +func (m *MockLogicalRouterPolicy) GetLogicalRouterPoliciesByExtID(lrName, key, value string) ([]*ovnnb.LogicalRouterPolicy, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLogicalRouterPoliciesByExtID", lrName, key, value) + ret0, _ := ret[0].([]*ovnnb.LogicalRouterPolicy) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLogicalRouterPoliciesByExtID indicates an expected call of GetLogicalRouterPoliciesByExtID. +func (mr *MockLogicalRouterPolicyMockRecorder) GetLogicalRouterPoliciesByExtID(lrName, key, value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogicalRouterPoliciesByExtID", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).GetLogicalRouterPoliciesByExtID), lrName, key, value) +} + // GetLogicalRouterPolicy mocks base method. func (m *MockLogicalRouterPolicy) GetLogicalRouterPolicy(lrName string, priority int, match string, ignoreNotFound bool) ([]*ovnnb.LogicalRouterPolicy, error) { m.ctrl.T.Helper() @@ -1913,18 +1942,18 @@ func (mr *MockLogicalRouterPolicyMockRecorder) GetLogicalRouterPolicy(lrName, pr } // ListLogicalRouterPolicies mocks base method. -func (m *MockLogicalRouterPolicy) ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) ([]*ovnnb.LogicalRouterPolicy, error) { +func (m *MockLogicalRouterPolicy) ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string, ignoreExtIDEmptyValue bool) ([]*ovnnb.LogicalRouterPolicy, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListLogicalRouterPolicies", lrName, priority, externalIDs) + ret := m.ctrl.Call(m, "ListLogicalRouterPolicies", lrName, priority, externalIDs, ignoreExtIDEmptyValue) ret0, _ := ret[0].([]*ovnnb.LogicalRouterPolicy) ret1, _ := ret[1].(error) return ret0, ret1 } // ListLogicalRouterPolicies indicates an expected call of ListLogicalRouterPolicies. -func (mr *MockLogicalRouterPolicyMockRecorder) ListLogicalRouterPolicies(lrName, priority, externalIDs interface{}) *gomock.Call { +func (mr *MockLogicalRouterPolicyMockRecorder) ListLogicalRouterPolicies(lrName, priority, externalIDs, ignoreExtIDEmptyValue interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLogicalRouterPolicies", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).ListLogicalRouterPolicies), lrName, priority, externalIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLogicalRouterPolicies", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).ListLogicalRouterPolicies), lrName, priority, externalIDs, ignoreExtIDEmptyValue) } // MockNAT is a mock of NAT interface. @@ -2890,6 +2919,20 @@ func (mr *MockNbClientMockRecorder) DeleteLogicalSwitchPort(lspName interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalSwitchPort", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalSwitchPort), lspName) } +// DeleteLogicalSwitchPorts mocks base method. +func (m *MockNbClient) DeleteLogicalSwitchPorts(externalIDs map[string]string, filter func(*ovnnb.LogicalSwitchPort) bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalSwitchPorts", externalIDs, filter) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalSwitchPorts indicates an expected call of DeleteLogicalSwitchPorts. +func (mr *MockNbClientMockRecorder) DeleteLogicalSwitchPorts(externalIDs, filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalSwitchPorts", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalSwitchPorts), externalIDs, filter) +} + // DeleteNat mocks base method. func (m *MockNbClient) DeleteNat(lrName, natType, externalIP, logicalIP string) error { m.ctrl.T.Helper() @@ -3020,6 +3063,21 @@ func (mr *MockNbClientMockRecorder) GetLogicalRouter(lrName, ignoreNotFound inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogicalRouter", reflect.TypeOf((*MockNbClient)(nil).GetLogicalRouter), lrName, ignoreNotFound) } +// GetLogicalRouterPoliciesByExtID mocks base method. +func (m *MockNbClient) GetLogicalRouterPoliciesByExtID(lrName, key, value string) ([]*ovnnb.LogicalRouterPolicy, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLogicalRouterPoliciesByExtID", lrName, key, value) + ret0, _ := ret[0].([]*ovnnb.LogicalRouterPolicy) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLogicalRouterPoliciesByExtID indicates an expected call of GetLogicalRouterPoliciesByExtID. +func (mr *MockNbClientMockRecorder) GetLogicalRouterPoliciesByExtID(lrName, key, value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogicalRouterPoliciesByExtID", reflect.TypeOf((*MockNbClient)(nil).GetLogicalRouterPoliciesByExtID), lrName, key, value) +} + // GetLogicalRouterPolicy mocks base method. func (m *MockNbClient) GetLogicalRouterPolicy(lrName string, priority int, match string, ignoreNotFound bool) ([]*ovnnb.LogicalRouterPolicy, error) { m.ctrl.T.Helper() @@ -3201,18 +3259,18 @@ func (mr *MockNbClientMockRecorder) ListLogicalRouter(needVendorFilter, filter i } // ListLogicalRouterPolicies mocks base method. -func (m *MockNbClient) ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) ([]*ovnnb.LogicalRouterPolicy, error) { +func (m *MockNbClient) ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string, ignoreExtIDEmptyValue bool) ([]*ovnnb.LogicalRouterPolicy, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListLogicalRouterPolicies", lrName, priority, externalIDs) + ret := m.ctrl.Call(m, "ListLogicalRouterPolicies", lrName, priority, externalIDs, ignoreExtIDEmptyValue) ret0, _ := ret[0].([]*ovnnb.LogicalRouterPolicy) ret1, _ := ret[1].(error) return ret0, ret1 } // ListLogicalRouterPolicies indicates an expected call of ListLogicalRouterPolicies. -func (mr *MockNbClientMockRecorder) ListLogicalRouterPolicies(lrName, priority, externalIDs interface{}) *gomock.Call { +func (mr *MockNbClientMockRecorder) ListLogicalRouterPolicies(lrName, priority, externalIDs, ignoreExtIDEmptyValue interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLogicalRouterPolicies", reflect.TypeOf((*MockNbClient)(nil).ListLogicalRouterPolicies), lrName, priority, externalIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLogicalRouterPolicies", reflect.TypeOf((*MockNbClient)(nil).ListLogicalRouterPolicies), lrName, priority, externalIDs, ignoreExtIDEmptyValue) } // ListLogicalRouterPorts mocks base method. diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 3615d6252c9..c777343f1a5 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -1239,7 +1239,7 @@ func (c *Controller) deletePolicyRouteForLocalDNSCacheOnNode(nodeName string, af "node": nodeName, "address-family": strconv.Itoa(af), "isLocalDnsCache": "true", - }) + }, true) if err != nil { klog.Errorf("failed to list logical router policies: %v", err) return err diff --git a/pkg/controller/ovn_ic.go b/pkg/controller/ovn_ic.go index e6765544687..1e59785dde3 100644 --- a/pkg/controller/ovn_ic.go +++ b/pkg/controller/ovn_ic.go @@ -18,6 +18,7 @@ import ( "k8s.io/klog/v2" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -165,15 +166,22 @@ func (c *Controller) removeInterConnection(azName string) error { } if azName != "" { - lspName := fmt.Sprintf("ts-%s", azName) - lrpName := fmt.Sprintf("%s-ts", azName) - if err := c.OVNNbClient.RemoveLogicalPatchPort(lspName, lrpName); err != nil { - klog.Errorf("delete ovn-ic logical port %s and %s: %v", lspName, lrpName, err) + if err := c.OVNNbClient.DeleteLogicalSwitchPorts(nil, func(lsp *ovnnb.LogicalSwitchPort) bool { + names := strings.Split(lsp.Name, "-") + return len(names) == 2 && names[1] == azName && strings.HasPrefix(names[0], "ts") + }); err != nil { + return err + } + + if err := c.OVNNbClient.DeleteLogicalRouterPorts(nil, func(lrp *ovnnb.LogicalRouterPort) bool { + names := strings.Split(lrp.Name, "-") + return len(names) == 2 && names[0] == azName && strings.HasPrefix(names[1], "ts") + }); err != nil { return err } } - if err := c.stopOvnIC(); err != nil { + if err := c.stopOVNIC(); err != nil { klog.Errorf("failed to stop ovn-ic, %v", err) return err } @@ -182,88 +190,91 @@ func (c *Controller) removeInterConnection(azName string) error { } func (c *Controller) establishInterConnection(config map[string]string) error { - if err := c.startOvnIC(config["ic-db-host"], config["ic-nb-port"], config["ic-sb-port"]); err != nil { + if err := c.startOVNIC(config["ic-db-host"], config["ic-nb-port"], config["ic-sb-port"]); err != nil { klog.Errorf("failed to start ovn-ic, %v", err) return err } - tsPort := fmt.Sprintf("ts-%s", config["az-name"]) - exist, err := c.OVNNbClient.LogicalSwitchPortExists(tsPort) - if err != nil { - klog.Errorf("failed to list logical switch ports, %v", err) - return err - } - - if exist { - klog.Infof("ts port %s already exists", tsPort) - return nil - } - if err := c.OVNNbClient.SetAzName(config["az-name"]); err != nil { klog.Errorf("failed to set az name. %v", err) return err } - chassises := []string{} - gwNodes := strings.Split(config["gw-nodes"], ",") - for _, gw := range gwNodes { - gw = strings.TrimSpace(gw) - cachedNode, err := c.nodesLister.Get(gw) + groups := strings.Split(config["gw-nodes"], ";") + for i, group := range groups { + gwNodes := strings.Split(group, ",") + chassises := make([]string, len(gwNodes)) + for j, gw := range gwNodes { + gw = strings.TrimSpace(gw) + chassis, err := c.OVNSbClient.GetChassisByHost(gw) + if err != nil { + klog.Errorf("failed to get gw %s chassis: %v", gw, err) + return err + } + if chassis.Name == "" { + return fmt.Errorf("no chassis for gw %s", gw) + } + chassises[j] = chassis.Name + + cachedNode, err := c.nodesLister.Get(gw) + if err != nil { + klog.Errorf("failed to get gw node %s, %v", gw, err) + return err + } + node := cachedNode.DeepCopy() + patchPayloadTemplate := `[{ + "op": "%s", + "path": "/metadata/labels", + "value": %s + }]` + op := "replace" + if len(node.Labels) == 0 { + op = "add" + } + node.Labels[util.ICGatewayLabel] = "true" + raw, _ := json.Marshal(node.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") + if err != nil { + klog.Errorf("patch gw node %s failed %v", gw, err) + return err + } + } + + tsName := c.getTSName(i) + cidr, err := c.getTSCidr(i) if err != nil { - klog.Errorf("failed to get gw node %s, %v", gw, err) + klog.Errorf("failed to get ts cidr index %d err: %v", i, err) return err } - node := cachedNode.DeepCopy() - patchPayloadTemplate := `[{ - "op": "%s", - "path": "/metadata/labels", - "value": %s - }]` - op := "replace" - if len(node.Labels) == 0 { - op = "add" + + if err := c.createTS(genHostAddress(config["ic-db-host"], config["ic-nb-port"]), tsName, cidr); err != nil { + klog.Errorf("failed to create ts %q: %v", tsName, err) + return err } - node.Labels[util.ICGatewayLabel] = "true" - raw, _ := json.Marshal(node.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") + + tsPort := fmt.Sprintf("%s-%s", tsName, config["az-name"]) + exist, err := c.OVNNbClient.LogicalSwitchPortExists(tsPort) if err != nil { - klog.Errorf("patch gw node %s failed %v", gw, err) + klog.Errorf("failed to list logical switch ports, %v", err) return err } - annoChassisName := node.Annotations[util.ChassisAnnotation] - if annoChassisName == "" { - err := fmt.Errorf("node %s has no chassis annotation, kube-ovn-cni not ready", gw) - klog.Error(err) - return err + if exist { + klog.Infof("ts port %s already exists", tsPort) + continue } - klog.Infof("gw node %s chassis %s", gw, annoChassisName) - chassis, err := c.OVNSbClient.GetChassis(annoChassisName, false) + + lrpAddr, err := c.acquireLrpAddress(tsName) if err != nil { - klog.Errorf("failed to get node chassis %s, %v", annoChassisName, err) + klog.Errorf("failed to acquire lrp address, %v", err) return err } - chassises = append(chassises, chassis.Name) - } - if len(chassises) == 0 { - klog.Error("no available ic gw") - return fmt.Errorf("no available ic gw") - } - if err := c.waitTsReady(); err != nil { - klog.Errorf("failed to wait ts ready, %v", err) - return err - } - lrpIP, err := c.acquireLrpAddress(util.InterconnectionSwitch) - if err != nil { - klog.Errorf("failed to acquire lrp address, %v", err) - return err - } - - lrpName := fmt.Sprintf("%s-ts", config["az-name"]) - if err := c.OVNNbClient.CreateLogicalPatchPort(util.InterconnectionSwitch, c.config.ClusterRouter, tsPort, lrpName, lrpIP, util.GenerateMac(), chassises...); err != nil { - klog.Errorf("failed to create ovn-ic lrp %v", err) - return err + lrpName := fmt.Sprintf("%s-%s", config["az-name"], tsName) + if err := c.OVNNbClient.CreateLogicalPatchPort(tsName, c.config.ClusterRouter, tsPort, lrpName, lrpAddr, util.GenerateMac(), chassises...); err != nil { + klog.Errorf("failed to create ovn-ic lrp %q: %v", lrpName, err) + return err + } } return nil @@ -272,12 +283,12 @@ func (c *Controller) establishInterConnection(config map[string]string) error { func (c *Controller) acquireLrpAddress(ts string) (string, error) { cidr, err := c.ovnLegacyClient.GetTsSubnet(ts) if err != nil { - klog.Errorf("failed to get ts subnet: %v", err) + klog.Errorf("failed to get ts subnet %s: %v", ts, err) return "", err } existAddress, err := c.listRemoteLogicalSwitchPortAddress() if err != nil { - klog.Errorf("list remote port address: %v", err) + klog.Errorf("failed to list remote port address, %v", err) return "", err } @@ -303,7 +314,7 @@ func (c *Controller) acquireLrpAddress(ts string) (string, error) { } } -func (c *Controller) startOvnIC(icHost, icNbPort, icSbPort string) error { +func (c *Controller) startOVNIC(icHost, icNbPort, icSbPort string) error { cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl", fmt.Sprintf("--ovn-ic-nb-db=%s", genHostAddress(icHost, icNbPort)), fmt.Sprintf("--ovn-ic-sb-db=%s", genHostAddress(icHost, icSbPort)), @@ -323,40 +334,39 @@ func (c *Controller) startOvnIC(icHost, icNbPort, icSbPort string) error { } output, err := cmd.CombinedOutput() if err != nil { - klog.Error(err) - return fmt.Errorf("%s", output) + return fmt.Errorf("output: %s, err: %v", output, err) } return nil } -func (c *Controller) stopOvnIC() error { +func (c *Controller) stopOVNIC() error { cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl", "stop_ic") output, err := cmd.CombinedOutput() if err != nil { - klog.Error(err) - return fmt.Errorf("%s", output) + return fmt.Errorf("output: %s, err: %v", output, err) } return nil } -func (c *Controller) waitTsReady() error { - retry := 6 - for retry > 0 { - ready, err := c.allSubnetReady(util.InterconnectionSwitch) - if err != nil { - klog.Error(err) - return err - } - - if ready { - return nil - } - - klog.Infof("wait for logical switch %s ready", util.InterconnectionSwitch) - time.Sleep(5 * time.Second) - retry-- +func (c *Controller) createTS(icNBAddr, tsName, subnet string) error { + cmd := exec.Command("ovn-ic-nbctl", + fmt.Sprintf("--db=%s", icNBAddr), + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + if os.Getenv("ENABLE_SSL") == "true" { + cmd = exec.Command("ovn-ic-nbctl", + fmt.Sprintf("--db=%s", icNBAddr), + "--private-key=/var/run/tls/key", + "--certificate=/var/run/tls/cert", + "--ca-cert=/var/run/tls/cacert", + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) } - return fmt.Errorf("timeout to wait for logical switch %s ready", util.InterconnectionSwitch) + return nil } func (c *Controller) delLearnedRoute() error { @@ -372,12 +382,16 @@ func (c *Controller) delLearnedRoute() error { return err } for _, r := range routeList { + var policy ovnnb.LogicalRouterStaticRoutePolicy + if r.Policy != nil { + policy = *r.Policy + } if err = c.deleteStaticRouteFromVpc( lr.Name, r.RouteTable, r.IPPrefix, r.Nexthop, - reversePolicy(*r.Policy), + reversePolicy(policy), ); err != nil { klog.Errorf("failed to delete learned static route %#v on logical router %s: %v", r, lr.Name, err) return err @@ -409,6 +423,8 @@ func genHostAddress(host, port string) (hostAddress string) { func (c *Controller) SynRouteToPolicy() { c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICConnected) c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICStatic) + // To support the version before kube-ovn v1.9, in which version the option tag is origin="" + c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICNone) } func (c *Controller) RemoveOldChassisInSbDB(azName string) error { @@ -459,6 +475,13 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { klog.Errorf("failed to list lr ovn-ic route %v", err) return } + + lrPolicyList, err := c.OVNNbClient.GetLogicalRouterPoliciesByExtID(lr.Name, key, value) + if err != nil { + klog.Errorf("failed to list ovn-ic lr policy: %v", err) + return + } + if len(lrRouteList) == 0 { klog.V(5).Info("lr ovn-ic route does not exist") err := c.OVNNbClient.DeleteLogicalRouterPolicies(lr.Name, util.OvnICPolicyPriority, map[string]string{key: value}) @@ -470,11 +493,7 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { } policyMap := map[string]string{} - lrPolicyList, err := c.OVNNbClient.ListLogicalRouterPolicies(lr.Name, util.OvnICPolicyPriority, map[string]string{key: value}) - if err != nil { - klog.Errorf("failed to list ovn-ic lr policy: %v", err) - return - } + for _, lrPolicy := range lrPolicyList { match, err := stripPrefix(lrPolicy.Match) if err != nil { @@ -483,42 +502,27 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { } policyMap[match] = lrPolicy.UUID } + networks := strset.NewWithSize(len(lrRouteList)) for _, lrRoute := range lrRouteList { - if _, ok := policyMap[lrRoute.IPPrefix]; ok { - delete(policyMap, lrRoute.IPPrefix) - } else { - var matchFiled string - if util.CheckProtocol(lrRoute.IPPrefix) == kubeovnv1.ProtocolIPv4 { - matchFiled = util.MatchV4Dst + " == " + lrRoute.IPPrefix - if err := c.addPolicyRouteToVpc( - lr.Name, - &kubeovnv1.PolicyRoute{ - Priority: util.OvnICPolicyPriority, - Match: matchFiled, - Action: kubeovnv1.PolicyRouteActionAllow, - }, - map[string]string{key: value, "vendor": util.CniTypeName}, - ); err != nil { - klog.Errorf("adding router policy failed %v", err) - } - } + networks.Add(lrRoute.IPPrefix) + } - if util.CheckProtocol(lrRoute.IPPrefix) == kubeovnv1.ProtocolIPv6 { - matchFiled = util.MatchV6Dst + " == " + lrRoute.IPPrefix - if err := c.addPolicyRouteToVpc( - lr.Name, - &kubeovnv1.PolicyRoute{ - Priority: util.OvnICPolicyPriority, - Match: matchFiled, - Action: kubeovnv1.PolicyRouteActionAllow, - }, - map[string]string{key: value, "vendor": util.CniTypeName}, - ); err != nil { - klog.Errorf("adding router policy failed %v", err) - } - } + networks.Each(func(prefix string) bool { + if _, ok := policyMap[prefix]; ok { + delete(policyMap, prefix) + return true } - } + match := util.MatchV4Dst + " == " + prefix + if util.CheckProtocol(prefix) == kubeovnv1.ProtocolIPv6 { + match = util.MatchV6Dst + " == " + prefix + } + + if err = c.OVNNbClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { + klog.Errorf("failed to add router policy: %v", err) + } + + return true + }) for _, uuid := range policyMap { if err := c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lr.Name, uuid); err != nil { klog.Errorf("deleting router policy failed %v", err) @@ -553,3 +557,28 @@ func (c *Controller) listRemoteLogicalSwitchPortAddress() (*strset.Set, error) { return existAddress, nil } + +func (c *Controller) getTSName(index int) string { + if index == 0 { + return util.InterconnectionSwitch + } + return fmt.Sprintf("%s%d", util.InterconnectionSwitch, index) +} + +func (c *Controller) getTSCidr(index int) (string, error) { + defaultSubnet, err := c.subnetsLister.Get(c.config.DefaultLogicalSwitch) + if err != nil { + return "", err + } + + var cidr string + switch defaultSubnet.Spec.Protocol { + case kubeovnv1.ProtocolIPv4: + cidr = fmt.Sprintf("169.254.%d.0/24", 100+index) + case kubeovnv1.ProtocolIPv6: + cidr = fmt.Sprintf("fe80:a9fe:%02x::/112", 100+index) + case kubeovnv1.ProtocolDual: + cidr = fmt.Sprintf("169.254.%d.0/24,fe80:a9fe:%02x::/112", 100+index, 100+index) + } + return cidr, nil +} diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index b21d16c855f..e671e1020ca 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -2669,7 +2669,7 @@ func (c *Controller) deletePolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet) "isU2ORoutePolicy": "true", "vendor": util.CniTypeName, "subnet": subnet.Name, - }) + }, true) if err != nil { klog.Errorf("failed to list logical router policies: %v", err) return err diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index d6bd1870156..4f18b547211 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -464,7 +464,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return err } } else { - policyRouteLogical, err = c.OVNNbClient.ListLogicalRouterPolicies(vpc.Name, -1, nil) + policyRouteLogical, err = c.OVNNbClient.ListLogicalRouterPolicies(vpc.Name, -1, nil, true) if err != nil { klog.Errorf("failed to get vpc %s policy route list, %v", vpc.Name, err) return err diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index 71fd519f2dd..5c8faa04ab0 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -74,6 +74,7 @@ type LogicalSwitchPort interface { SetLogicalSwitchPortsSecurityGroup(sgName, op string) error EnablePortLayer2forward(lspName string) error DeleteLogicalSwitchPort(lspName string) error + DeleteLogicalSwitchPorts(externalIDs map[string]string, filter func(lsp *ovnnb.LogicalSwitchPort) bool) error ListLogicalSwitchPorts(needVendorFilter bool, externalIDs map[string]string, filter func(lsp *ovnnb.LogicalSwitchPort) bool) ([]ovnnb.LogicalSwitchPort, error) ListNormalLogicalSwitchPorts(needVendorFilter bool, externalIDs map[string]string) ([]ovnnb.LogicalSwitchPort, error) ListLogicalSwitchPortsWithLegacyExternalIDs() ([]ovnnb.LogicalSwitchPort, error) @@ -157,8 +158,9 @@ type LogicalRouterPolicy interface { DeleteLogicalRouterPolicyByUUID(lrName, uuid string) error DeleteLogicalRouterPolicyByNexthop(lrName string, priority int, nexthop string) error ClearLogicalRouterPolicy(lrName string) error - ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) ([]*ovnnb.LogicalRouterPolicy, error) + ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string, ignoreExtIDEmptyValue bool) ([]*ovnnb.LogicalRouterPolicy, error) GetLogicalRouterPolicy(lrName string, priority int, match string, ignoreNotFound bool) ([]*ovnnb.LogicalRouterPolicy, error) + GetLogicalRouterPoliciesByExtID(lrName, key, value string) ([]*ovnnb.LogicalRouterPolicy, error) } type NAT interface { diff --git a/pkg/ovs/ovn-nb-logical_router_policy.go b/pkg/ovs/ovn-nb-logical_router_policy.go index db95486c338..fc40b31b84b 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy.go +++ b/pkg/ovs/ovn-nb-logical_router_policy.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "github.com/ovn-org/libovsdb/client" "github.com/ovn-org/libovsdb/model" @@ -29,7 +30,7 @@ func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, var found bool duplicate := make([]string, 0, len(policyList)) for _, policy := range policyList { - if found || policy.Action != action || (policy.Action == ovnnb.LogicalRouterPolicyActionReroute && !strset.New(nextHops...).IsEqual(strset.New(policy.Nexthops...))) { + if found || policy.Action != action || !reflect.DeepEqual(policy.ExternalIDs, externalIDs) || (policy.Action == ovnnb.LogicalRouterPolicyActionReroute && !strset.New(nextHops...).IsEqual(strset.New(policy.Nexthops...))) { duplicate = append(duplicate, policy.UUID) } else { found = true @@ -109,7 +110,7 @@ func (c *OVNNbClient) DeleteLogicalRouterPolicy(lrName string, priority int, mat // DeleteLogicalRouterPolicy delete some policies from logical router once func (c *OVNNbClient) DeleteLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) error { // remove policies from logical router - policies, err := c.ListLogicalRouterPolicies(lrName, priority, externalIDs) + policies, err := c.ListLogicalRouterPolicies(lrName, priority, externalIDs, false) if err != nil { klog.Error(err) return err @@ -224,16 +225,22 @@ func (c *OVNNbClient) GetLogicalRouterPolicyByUUID(uuid string) (*ovnnb.LogicalR return policy, nil } +// GetLogicalRouterPoliciesByExtID get logical router policy route by external ID func (c *OVNNbClient) GetLogicalRouterPoliciesByExtID(lrName, key, value string) ([]*ovnnb.LogicalRouterPolicy, error) { fnFilter := func(policy *ovnnb.LogicalRouterPolicy) bool { - return len(policy.ExternalIDs) != 0 && policy.ExternalIDs[key] == value + if len(policy.ExternalIDs) != 0 { + if _, ok := policy.ExternalIDs[key]; ok { + return policy.ExternalIDs[key] == value + } + } + return false } return c.listLogicalRouterPoliciesByFilter(lrName, fnFilter) } // ListLogicalRouterPolicies list route policy which match the given externalIDs -func (c *OVNNbClient) ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) ([]*ovnnb.LogicalRouterPolicy, error) { - return c.listLogicalRouterPoliciesByFilter(lrName, policyFilter(priority, externalIDs)) +func (c *OVNNbClient) ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string, ignoreExtIDEmptyValue bool) ([]*ovnnb.LogicalRouterPolicy, error) { + return c.listLogicalRouterPoliciesByFilter(lrName, policyFilter(priority, externalIDs, ignoreExtIDEmptyValue)) } // newLogicalRouterPolicy return logical router policy with basic information @@ -249,7 +256,7 @@ func (c *OVNNbClient) newLogicalRouterPolicy(priority int, match, action string, } // policyFilter filter policies which match the given externalIDs -func policyFilter(priority int, externalIDs map[string]string) func(policy *ovnnb.LogicalRouterPolicy) bool { +func policyFilter(priority int, externalIDs map[string]string, ignoreExtIDEmptyValue bool) func(policy *ovnnb.LogicalRouterPolicy) bool { return func(policy *ovnnb.LogicalRouterPolicy) bool { if len(policy.ExternalIDs) < len(externalIDs) { return false @@ -257,9 +264,10 @@ func policyFilter(priority int, externalIDs map[string]string) func(policy *ovnn if len(policy.ExternalIDs) != 0 { for k, v := range externalIDs { + // ignoreExtIDEmptyValue is used to the case below: // if only key exist but not value in externalIDs, we should include this lsp, // it's equal to shell command `ovn-nbctl --columns=xx find logical_router_policy external_ids:key!=\"\"` - if len(v) == 0 { + if len(v) == 0 && ignoreExtIDEmptyValue { if len(policy.ExternalIDs[k]) == 0 { return false } diff --git a/pkg/ovs/ovn-nb-logical_router_policy_test.go b/pkg/ovs/ovn-nb-logical_router_policy_test.go index c03cc90d70a..d3ea5585040 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy_test.go +++ b/pkg/ovs/ovn-nb-logical_router_policy_test.go @@ -159,7 +159,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicies() { require.NoError(t, err) require.Len(t, lr.Policies, 3) - policies, err := ovnClient.ListLogicalRouterPolicies(lrName, -1, externalIDs) + policies, err := ovnClient.ListLogicalRouterPolicies(lrName, -1, externalIDs, true) require.NoError(t, err) require.Len(t, policies, 3) } @@ -174,7 +174,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicies() { require.NoError(t, err) require.Empty(t, lr.Policies) - policies, err := ovnClient.ListLogicalRouterPolicies(lrName, -1, externalIDs) + policies, err := ovnClient.ListLogicalRouterPolicies(lrName, -1, externalIDs, true) require.NoError(t, err) require.Empty(t, policies) }) @@ -190,7 +190,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicies() { require.Len(t, lr.Policies, 2) // no basePriority policy - policies, err := ovnClient.ListLogicalRouterPolicies(lrName, -1, externalIDs) + policies, err := ovnClient.ListLogicalRouterPolicies(lrName, -1, externalIDs, true) require.NoError(t, err) require.Len(t, policies, 2) }) @@ -342,7 +342,7 @@ func (suite *OvnClientTestSuite) testPolicyFilter() { } t.Run("include all policies", func(t *testing.T) { - filterFunc := policyFilter(-1, nil) + filterFunc := policyFilter(-1, nil, true) count := 0 for _, policy := range policies { if filterFunc(policy) { @@ -353,7 +353,7 @@ func (suite *OvnClientTestSuite) testPolicyFilter() { }) t.Run("include all policies with external ids", func(t *testing.T) { - filterFunc := policyFilter(-1, map[string]string{"k1": "v1"}) + filterFunc := policyFilter(-1, map[string]string{"k1": "v1"}, true) count := 0 for _, policy := range policies { if filterFunc(policy) { @@ -364,7 +364,7 @@ func (suite *OvnClientTestSuite) testPolicyFilter() { }) t.Run("include all policies with same priority", func(t *testing.T) { - filterFunc := policyFilter(10000, map[string]string{"k1": "v1"}) + filterFunc := policyFilter(10000, map[string]string{"k1": "v1"}, true) count := 0 for _, policy := range policies { if filterFunc(policy) { @@ -381,7 +381,7 @@ func (suite *OvnClientTestSuite) testPolicyFilter() { filterFunc := policyFilter(-1, map[string]string{ "k1": "v1", "key": "value", - }) + }, true) require.False(t, filterFunc(policy)) }) } diff --git a/pkg/ovs/ovn-nb-logical_router_route.go b/pkg/ovs/ovn-nb-logical_router_route.go index 0f229162de5..290b3cdeda9 100644 --- a/pkg/ovs/ovn-nb-logical_router_route.go +++ b/pkg/ovs/ovn-nb-logical_router_route.go @@ -18,7 +18,12 @@ import ( func (c *OVNNbClient) ListLogicalRouterStaticRoutesByOption(lrName, _, key, value string) ([]*ovnnb.LogicalRouterStaticRoute, error) { fnFilter := func(route *ovnnb.LogicalRouterStaticRoute) bool { - return len(route.Options) != 0 && route.Options[key] == value + if len(route.Options) != 0 { + if _, ok := route.Options[key]; ok { + return route.Options[key] == value + } + } + return false } return c.listLogicalRouterStaticRoutesByFilter(lrName, fnFilter) } diff --git a/pkg/ovs/ovn-nb-logical_switch_port.go b/pkg/ovs/ovn-nb-logical_switch_port.go index 92488bd5870..83fe2d69a39 100644 --- a/pkg/ovs/ovn-nb-logical_switch_port.go +++ b/pkg/ovs/ovn-nb-logical_switch_port.go @@ -564,6 +564,31 @@ func (c *OVNNbClient) DeleteLogicalSwitchPort(lspName string) error { return nil } +// DeleteLogicalSwitchPorts delete logical switch port from logical switch +func (c *OVNNbClient) DeleteLogicalSwitchPorts(externalIDs map[string]string, filter func(lrp *ovnnb.LogicalSwitchPort) bool) error { + lspList, err := c.ListLogicalSwitchPorts(false, externalIDs, filter) + if err != nil { + klog.Error(err) + return fmt.Errorf("list switch ports: %v", err) + } + + ops := make([]ovsdb.Operation, 0, len(lspList)) + for _, lsp := range lspList { + op, err := c.DeleteLogicalSwitchPortOp(lsp.Name) + if err != nil { + klog.Error(err) + return fmt.Errorf("generate operations for deleting logical switch port %s: %v", lsp.Name, err) + } + ops = append(ops, op...) + } + + if err := c.Transact("lsps-del", ops); err != nil { + return fmt.Errorf("del logical switch ports: %v", err) + } + + return nil +} + // GetLogicalSwitchPort get logical switch port by name func (c *OVNNbClient) GetLogicalSwitchPort(lspName string, ignoreNotFound bool) (*ovnnb.LogicalSwitchPort, error) { ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) diff --git a/pkg/util/const.go b/pkg/util/const.go index 933c879dd94..6c96b15a265 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -236,6 +236,7 @@ const ( OvnICKey = "origin" OvnICConnected = "connected" OvnICStatic = "static" + OvnICNone = "" MatchV4Src = "ip4.src" MatchV4Dst = "ip4.dst" diff --git a/test/e2e/ovn-ic/e2e_test.go b/test/e2e/ovn-ic/e2e_test.go index 6328c1d9ec6..df2e3dd763a 100644 --- a/test/e2e/ovn-ic/e2e_test.go +++ b/test/e2e/ovn-ic/e2e_test.go @@ -7,6 +7,7 @@ import ( "fmt" "math/rand" "net" + "os/exec" "strconv" "strings" "testing" @@ -206,4 +207,80 @@ var _ = framework.OrderedDescribe("[group:ovn-ic]", func() { fnCheckPodHTTP() }) + + framework.ConformanceIt("should be able to update gateway to ecmp or HA ", func() { + frameworks[0].SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + gwNodes := make([]string, len(clusters)) + for i := range clusters { + ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) + cm, err := clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.TODO(), util.InterconnectionConfig, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get ConfigMap") + gwNodes[i] = cm.Data["gw-nodes"] + } + + ginkgo.By("Case 1: Changing the ConfigMap in cluster to HA") + changeGatewayType("ha", gwNodes, clientSets) + ginkgo.By("Waiting for HA gateway to be applied") + time.Sleep(15 * time.Second) + + checkECMPCount(0) + fnCheckPodHTTP() + + ginkgo.By("Case 2: Changing the ConfigMap in cluster to ecmp ") + changeGatewayType("ecmp", gwNodes, clientSets) + ginkgo.By("Waiting for ecmp gateway to be applied") + time.Sleep(15 * time.Second) + if frameworks[0].ClusterIPFamily == "dual" { + checkECMPCount(6) + } else { + checkECMPCount(3) + } + fnCheckPodHTTP() + + ginkgo.By("Case 3: Changing the ConfigMap in cluster to ha + ecmp") + changeGatewayType("half", gwNodes, clientSets) + ginkgo.By("Waiting for half gateway to be applied") + time.Sleep(15 * time.Second) + + if frameworks[0].ClusterIPFamily == "dual" { + checkECMPCount(4) + } else { + checkECMPCount(2) + } + fnCheckPodHTTP() + }) }) + +func checkECMPCount(expectCount int) { + execCmd := "kubectl ko nbctl lr-route-list ovn-cluster " + output, err := exec.Command("bash", "-c", execCmd).CombinedOutput() + ecmpCount := strings.Count(string(output), "ecmp") + framework.ExpectNoError(err) + framework.ExpectEqual(ecmpCount, expectCount) +} + +func changeGatewayType(gatewayType string, gwNodes []string, clientSets []clientset.Interface) { + for index, clientSet := range clientSets { + var gatewayStr string + switch gatewayType { + case "ha": + gatewayStr = strings.ReplaceAll(gwNodes[index], ";", ",") + case "ecmp": + gatewayStr = strings.ReplaceAll(gwNodes[index], ",", ";") + case "half": + gatewayStr = gwNodes[index] + } + framework.Logf("check gatewayStr %s ", gatewayStr) + configMapPatchPayload, err := json.Marshal(corev1.ConfigMap{ + Data: map[string]string{ + "gw-nodes": gatewayStr, + }, + }) + + framework.ExpectNoError(err, "failed to marshal patch data") + + ginkgo.By("patching the ConfigMap in cluster " + clusters[index]) + _, err = clientSet.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, configMapPatchPayload, metav1.PatchOptions{}) + framework.ExpectNoError(err, "failed to patch ConfigMap") + } +} diff --git a/yamls/kind.yaml.j2 b/yamls/kind.yaml.j2 index 74d73b1dca7..81fcd33eb61 100644 --- a/yamls/kind.yaml.j2 +++ b/yamls/kind.yaml.j2 @@ -10,6 +10,9 @@ {%- if ha is not defined -%} {%- set ha = "false" -%} {%- endif -%} +{%- if ovn_ic is not defined -%} + {%- set ovn_ic = "false" -%} +{%- endif -%} {%- if single is not defined -%} {%- set single = "false" -%} {%- endif -%} @@ -78,5 +81,8 @@ nodes: image: kindest/node:{{ k8s_version }} labels: kube-ovn/role: master + {%- elif ovn_ic is equalto "true" %} + - role: worker + image: kindest/node:{{ k8s_version }} {%- endif %} {%- endif %}