diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 015de894f7f..ee7a151e8c1 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -1675,6 +1675,13 @@ jobs: - build-e2e-binaries runs-on: ubuntu-22.04 timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + ip-family: + - ipv4 + # - ipv6 + # - dual steps: - uses: jlumbroso/free-disk-space@v1.3.1 with: @@ -1757,18 +1764,19 @@ jobs: run: | sudo pip3 install j2cli sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init + sudo PATH=~/.local/bin:$PATH make kind-init-cilium-chaining-${{ matrix.ip-family }} sudo cp -r /root/.kube/ ~/.kube/ sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN with Cilium chaining - run: make kind-install-cilium-chaining + run: make kind-install-cilium-chaining-${{ matrix.ip-family }} - name: Run E2E working-directory: ${{ env.E2E_DIR }} env: E2E_CILIUM_CHAINING: "true" E2E_BRANCH: ${{ github.base_ref || github.ref_name }} + E2E_IP_FAMILY: ${{ matrix.ip-family }} run: make k8s-conformance-e2e - name: kubectl ko log diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml index a74cb7d49dc..ab35419c308 100644 --- a/.github/workflows/scheduled-e2e.yaml +++ b/.github/workflows/scheduled-e2e.yaml @@ -896,6 +896,7 @@ jobs: run: | sudo pip3 install j2cli sudo pip3 install "j2cli[yaml]" + sudo PATH=~/.local/bin:$PATH make kind-init-cilium-chaining || \ sudo PATH=~/.local/bin:$PATH make kind-init sudo cp -r /root/.kube/ ~/.kube/ sudo chown -R $(id -un). ~/.kube/ diff --git a/Makefile b/Makefile index b6421a03bc5..bb4316b9c18 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,8 @@ KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/ KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-cr.yaml KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml -CILIUM_VERSION = 1.14.7 -CILIUM_IMAGE_REPO = quay.io/cilium/cilium +CILIUM_VERSION = 1.15.1 +CILIUM_IMAGE_REPO = quay.io/cilium CERT_MANAGER_VERSION = v1.14.2 CERT_MANAGER_CONTROLLER = quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION) @@ -349,31 +349,26 @@ kind-create: .PHONY: kind-init kind-init: kind-init-ipv4 -.PHONY: kind-init-ipv4 -kind-init-ipv4: kind-clean - @$(MAKE) kind-generate-config +.PHONY: kind-init-% +kind-init-%: kind-clean + @ip_family=$* $(MAKE) kind-generate-config @$(MAKE) kind-create .PHONY: kind-init-ovn-ic kind-init-ovn-ic: kind-init-ovn-ic-ipv4 -.PHONY: kind-init-ovn-ic-ipv4 -kind-init-ovn-ic-ipv4: kind-clean-ovn-ic - @ha=true $(MAKE) kind-init - @ovn_ic=true $(MAKE) kind-generate-config +.PHONY: kind-init-ovn-ic-% +kind-init-ovn-ic-%: kind-clean-ovn-ic + @ha=true $(MAKE) kind-init-$* + @ovn_ic=true ip_family=$* $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) -.PHONY: kind-init-ovn-ic-ipv6 -kind-init-ovn-ic-ipv6: kind-clean-ovn-ic - @ha=true $(MAKE) kind-init-ipv6 - @ovn_ic=true ip_family=ipv6 $(MAKE) kind-generate-config - $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) +.PHONY: kind-init-cilium-chaining +kind-init-cilium-chaining: kind-init-cilium-chaining-ipv4 -.PHONY: kind-init-ovn-ic-dual -kind-init-ovn-ic-dual: kind-clean-ovn-ic - @ha=true $(MAKE) kind-init-dual - @ovn_ic=true ip_family=dual $(MAKE) kind-generate-config - $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) +.PHONY: kind-init-cilium-chaining-% +kind-init-cilium-chaining-%: + @kube_proxy_mode=none $(MAKE) kind-init-$* .PHONY: kind-init-ovn-submariner kind-init-ovn-submariner: kind-clean-ovn-submariner kind-init @@ -392,29 +387,16 @@ kind-init-iptables: .PHONY: kind-init-ha kind-init-ha: kind-init-ha-ipv4 -.PHONY: kind-init-ha-ipv4 -kind-init-ha-ipv4: - @ha=true $(MAKE) kind-init - -.PHONY: kind-init-ha-ipv6 -kind-init-ha-ipv6: - @ip_family=ipv6 $(MAKE) kind-init-ha - -.PHONY: kind-init-ha-dual -kind-init-ha-dual: - @ip_family=dual $(MAKE) kind-init-ha +.PHONY: kind-init-ha-% +kind-init-ha-%: + @ha=true $(MAKE) kind-init-$* .PHONY: kind-init-single -kind-init-single: - @single=true $(MAKE) kind-init +kind-init-single: kind-init-single-ipv4 -.PHONY: kind-init-ipv6 -kind-init-ipv6: - @ip_family=ipv6 $(MAKE) kind-init - -.PHONY: kind-init-dual -kind-init-dual: - @ip_family=dual $(MAKE) kind-init +.PHONY: kind-init-single-% +kind-init-single-%: + @single=true $(MAKE) kind-init-$* .PHONY: kind-init-bgp kind-init-bgp: kind-clean-bgp kind-init @@ -502,23 +484,42 @@ kind-install: kind-load-image sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install.sh | bash kubectl describe no +.PHONY: kind-install-ipv4 +kind-install-ipv4: kind-install + +.PHONY: kind-install-ipv6 +kind-install-ipv6: + @IPV6=true $(MAKE) kind-install + +.PHONY: kind-install-dual +kind-install-dual: + @DUAL_STACK=true $(MAKE) kind-install + +.PHONY: kind-install-overlay-% +kind-install-overlay-%: + @$(MAKE) kind-install-$* + .PHONY: kind-install-dev -kind-install-dev: - @VERSION=$(DEV_TAG) $(MAKE) kind-install +kind-install-dev: kind-install-dev-ipv4 + +.PHONY: kind-install-dev-% +kind-install-dev-%: + @VERSION=$(DEV_TAG) $(MAKE) kind-install-$* .PHONY: kind-install-debug -kind-install-debug: - @VERSION=$(DEBUG_TAG) $(MAKE) kind-install +kind-install-debug: kind-install-debug-ipv4 + +.PHONY: kind-install-debug-% +kind-install-debug-%: + @VERSION=$(DEBUG_TAG) $(MAKE) kind-install-$* .PHONY: kind-install-debug-valgrind -kind-install-debug-valgrind: +kind-install-debug-valgrind: kind-install-debug-valgrind-ipv4 @DEBUG_WRAPPER=valgrind $(MAKE) kind-install-debug -.PHONY: kind-install-ipv4 -kind-install-ipv4: kind-install-overlay-ipv4 - -.PHONY: kind-install-overlay-ipv4 -kind-install-overlay-ipv4: kind-install +.PHONY: kind-install-debug-valgrind-% +kind-install-debug-valgrind-%: + @DEBUG_WRAPPER=valgrind $(MAKE) kind-install-debug-$* .PHONY: kind-install-ovn-ic kind-install-ovn-ic: kind-install-ovn-ic-ipv4 @@ -668,13 +669,6 @@ kind-install-underlay-hairpin-ipv4: kind-enable-hairpin kind-load-image kind-unt ENABLE_VLAN=true VLAN_NIC=eth0 bash kubectl describe no -.PHONY: kind-install-ipv6 -kind-install-ipv6: kind-install-overlay-ipv6 - -.PHONY: kind-install-overlay-ipv6 -kind-install-overlay-ipv6: - @IPV6=true $(MAKE) kind-install - .PHONY: kind-install-underlay-ipv6 kind-install-underlay-ipv6: kind-disable-hairpin kind-load-image kind-untaint-control-plane $(call docker_network_info,kind) @@ -697,13 +691,6 @@ kind-install-underlay-hairpin-ipv6: kind-enable-hairpin kind-load-image kind-unt dist/images/install.sh | \ IPV6=true ENABLE_VLAN=true VLAN_NIC=eth0 bash -.PHONY: kind-install-dual -kind-install-dual: kind-install-overlay-dual - -.PHONY: kind-install-overlay-dual -kind-install-overlay-dual: - @DUAL_STACK=true $(MAKE) kind-install - .PHONY: kind-install-underlay-dual kind-install-underlay-dual: kind-disable-hairpin kind-load-image kind-untaint-control-plane $(call docker_network_info,kind) @@ -793,27 +780,44 @@ kind-install-webhook: kind-install kubectl rollout status deployment/kube-ovn-webhook -n kube-system --timeout 120s .PHONY: kind-install-cilium-chaining -kind-install-cilium-chaining: kind-load-image kind-untaint-control-plane +kind-install-cilium-chaining: kind-install-cilium-chaining-ipv4 + +.PHONY: kind-install-cilium-chaining-% +kind-install-cilium-chaining-%: $(eval KUBERNETES_SERVICE_HOST = $(shell kubectl get nodes kube-ovn-control-plane -o jsonpath='{.status.addresses[0].address}')) - $(call kind_load_image,kube-ovn,$(CILIUM_IMAGE_REPO):v$(CILIUM_VERSION),1) + $(call kind_load_image,kube-ovn,$(CILIUM_IMAGE_REPO)/cilium:v$(CILIUM_VERSION),1) + $(call kind_load_image,kube-ovn,$(CILIUM_IMAGE_REPO)/operator-generic:v$(CILIUM_VERSION),1) kubectl apply -f yamls/cilium-chaining.yaml helm repo add cilium https://helm.cilium.io/ helm repo update helm install cilium cilium/cilium \ --version $(CILIUM_VERSION) \ - --namespace=kube-system \ + --namespace kube-system \ --set k8sServiceHost=$(KUBERNETES_SERVICE_HOST) \ --set k8sServicePort=6443 \ - --set tunnel=disabled \ + --set kubeProxyReplacement=partial \ + --set socketLB.enabled=true \ + --set nodePort.enabled=true \ + --set externalIPs.enabled=true \ + --set hostPort.enabled=false \ + --set routingMode=native \ --set sessionAffinity=true \ --set enableIPv4Masquerade=false \ + --set enableIPv6Masquerade=false \ + --set hubble.enabled=true \ + --set sctp.enabled=true \ + --set ipv4.enabled=$(shell [ $* = ipv6 ] && echo false || echo true) \ + --set ipv6.enabled=$(shell [ $* = ipv4 ] && echo false || echo true) \ + --set k8s.requireIPv4PodCIDR=$(shell [ $* = ipv6 ] && echo false || echo true) \ + --set k8s.requireIPv6PodCIDR=$(shell [ $* = ipv4 ] && echo false || echo true) \ --set cni.chainingMode=generic-veth \ + --set cni.chainingTarget=kube-ovn \ --set cni.customConf=true \ --set cni.configMap=cni-configuration kubectl -n kube-system rollout status ds cilium --timeout 300s - bash dist/images/install-cilium-cli.sh - sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install.sh | \ - ENABLE_LB=false ENABLE_NP=false CNI_CONFIG_PRIORITY=10 bash + @$(MAKE) ENABLE_LB=false ENABLE_NP=false \ + CNI_CONFIG_PRIORITY=10 WITHOUT_KUBE_PROXY=true \ + kind-install-$* kubectl describe no .PHONY: kind-install-bgp diff --git a/Makefile.e2e b/Makefile.e2e index 6e22cb7d93b..8b54e3329ec 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -37,10 +37,16 @@ endif ifeq ($(E2E_IP_FAMILY),dual) K8S_CONFORMANCE_E2E_FOCUS += "sig-network.*Feature:IPv6DualStack" +K8S_CONFORMANCE_E2E_SKIP += "sig-network.*should create pod, add ipv6 and ipv4 ip to host ips.*Feature:PodHostIPs" endif ifeq ($(E2E_CILIUM_CHAINING),true) +# https://docs.cilium.io/en/stable/configuration/sctp/ +# SCTP support does not support rewriting ports for SCTP packets. +# This means that when defining services, the targetPort MUST equal the port, +# otherwise the packet will be dropped. K8S_CONFORMANCE_E2E_SKIP += "sig-network.*Networking.*Feature:SCTPConnectivity" +# https://github.com/cilium/cilium/issues/9207 K8S_CONFORMANCE_E2E_SKIP += "sig-network.*Services.*should serve endpoints on same port and different protocols" endif diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 1e8eb922194..d4cddedadb7 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -56,8 +56,10 @@ func NewDefaultFramework(baseName string) *Framework { if strings.HasPrefix(f.ClusterVersion, "release-") { n, err := fmt.Sscanf(f.ClusterVersion, "release-%d.%d", &f.ClusterVersionMajor, &f.ClusterVersionMinor) - ExpectNoError(err) - ExpectEqual(n, 2) + if err != nil || n != 2 { + defer ginkgo.GinkgoRecover() + ginkgo.Fail(fmt.Sprintf("Failed to parse Kube-OVN version string %q", f.ClusterVersion)) + } } else { f.ClusterVersionMajor, f.ClusterVersionMinor = 999, 999 } @@ -104,8 +106,10 @@ func NewFrameworkWithContext(baseName, kubeContext string) *Framework { if strings.HasPrefix(f.ClusterVersion, "release-") { n, err := fmt.Sscanf(f.ClusterVersion, "release-%d.%d", &f.ClusterVersionMajor, &f.ClusterVersionMinor) - ExpectNoError(err) - ExpectEqual(n, 2) + if err != nil || n != 2 { + defer ginkgo.GinkgoRecover() + ginkgo.Fail(fmt.Sprintf("Failed to parse Kube-OVN version string %q", f.ClusterVersion)) + } } else { f.ClusterVersionMajor, f.ClusterVersionMinor = 999, 999 } @@ -192,7 +196,7 @@ func OrderedDescribe(text string, body func()) bool { // ConformanceIt is wrapper function for ginkgo It. // Adds "[Conformance]" tag and makes static analysis easier. func ConformanceIt(text string, body interface{}) bool { - return framework.ConformanceIt(text, body, framework.WithConformance()) + return framework.ConformanceIt(text, body) } func DisruptiveIt(text string, body interface{}) bool { diff --git a/yamls/cilium-chaining.yaml b/yamls/cilium-chaining.yaml index 402ef19e4c9..fe26c51b564 100644 --- a/yamls/cilium-chaining.yaml +++ b/yamls/cilium-chaining.yaml @@ -11,22 +11,18 @@ data: "plugins": [ { "type": "kube-ovn", - "log_level": "info", - "datastore_type": "kubernetes", - "mtu": 1400, - "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", - "ipam": { - "type": "kube-ovn", - "server_socket": "/run/openvswitch/kube-ovn-daemon.sock" - } + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock" }, { "type": "portmap", "snat": true, - "capabilities": {"portMappings": true} + "capabilities": { + "portMappings": true + } }, { - "type": "cilium-cni" + "type": "cilium-cni", + "chaining-mode": "generic-veth" } ] }