diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 31d7f29b424..76713bb25c5 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -915,7 +915,10 @@ jobs: E2E_BRANCH: ${{ github.base_ref || github.ref_name }} E2E_IP_FAMILY: ${{ matrix.ip-family }} E2E_NETWORK_MODE: ${{ matrix.mode }} - run: make kube-ovn-conformance-e2e + run: | + make kube-ovn-conformance-e2e + make kind-install-kubevirt + make kube-ovn-kubevirt-e2e - name: kubectl ko log if: failure() @@ -1261,101 +1264,6 @@ jobs: E2E_BRANCH: ${{ github.base_ref || github.ref_name }} run: make kube-ovn-lb-svc-conformance-e2e - kubevirt-e2e: - name: Kubevirt VM E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 15 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download kube-ovn image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load images - run: | - docker load -i kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN and KubeVirt - run: make kind-install-kubevirt - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - run: make kube-ovn-kubevirt-e2e - webhook-e2e: name: Webhook E2E needs: @@ -2103,7 +2011,6 @@ jobs: - no-np-test - cilium-chaining-e2e - kube-ovn-ha-e2e - - kubevirt-e2e - kube-ovn-submariner-conformance-e2e if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') runs-on: ubuntu-22.04 diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml index 4f395ad2203..555bb277f20 100644 --- a/.github/workflows/scheduled-e2e.yaml +++ b/.github/workflows/scheduled-e2e.yaml @@ -627,6 +627,7 @@ jobs: run: | version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"') docker pull kubeovn/kube-ovn:$version + VERSION=$version make kind-install VERSION=$version make kind-install-kubevirt - name: Run E2E diff --git a/Makefile b/Makefile index 6be37dff259..7a93a5f8d03 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ endif CONTROL_PLANE_TAINTS = node-role.kubernetes.io/master node-role.kubernetes.io/control-plane -FRR_VERSION = 8.5.4 +FRR_VERSION = 9.0.2 FRR_IMAGE = quay.io/frrouting/frr:$(FRR_VERSION) CLAB_IMAGE = ghcr.io/srl-labs/clab:0.54.2 @@ -46,10 +46,8 @@ KUBEVIRT_API_IMAGE = quay.io/kubevirt/virt-api:$(KUBEVIRT_VERSION) KUBEVIRT_CONTROLLER_IMAGE = quay.io/kubevirt/virt-controller:$(KUBEVIRT_VERSION) KUBEVIRT_HANDLER_IMAGE = quay.io/kubevirt/virt-handler:$(KUBEVIRT_VERSION) KUBEVIRT_LAUNCHER_IMAGE = quay.io/kubevirt/virt-launcher:$(KUBEVIRT_VERSION) -KUBEVIRT_TEST_IMAGE = quay.io/kubevirt/cirros-container-disk-demo KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-operator.yaml KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-cr.yaml -KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml CILIUM_VERSION = 1.15.3 CILIUM_IMAGE_REPO = quay.io/cilium @@ -744,7 +742,7 @@ kind-install-metallb: kind-install --version $(METALLB_VERSION) \ --namespace metallb-system \ --create-namespace \ - --set frr.image.tag=$(FRR_VERSION) + --set speaker.frr.image.tag=$(FRR_VERSION) $(call kubectl_wait_exist_and_ready,metallb-system,deployment,metallb-controller) $(call kubectl_wait_exist_and_ready,metallb-system,daemonset,metallb-speaker) @metallb_pool=$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.201/')-$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.250/') \ @@ -758,26 +756,23 @@ kind-install-vpc-nat-gw: @$(MAKE) kind-install-multus .PHONY: kind-install-kubevirt -kind-install-kubevirt: kind-install +kind-install-kubevirt: $(call kind_load_image,kube-ovn,$(KUBEVIRT_OPERATOR_IMAGE),1) $(call kind_load_image,kube-ovn,$(KUBEVIRT_API_IMAGE),1) $(call kind_load_image,kube-ovn,$(KUBEVIRT_CONTROLLER_IMAGE),1) $(call kind_load_image,kube-ovn,$(KUBEVIRT_HANDLER_IMAGE),1) $(call kind_load_image,kube-ovn,$(KUBEVIRT_LAUNCHER_IMAGE),1) - $(call kind_load_image,kube-ovn,$(KUBEVIRT_TEST_IMAGE),1) kubectl apply -f "$(KUBEVIRT_OPERATOR_YAML)" kubectl apply -f "$(KUBEVIRT_CR_YAML)" - kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}' + kubectl -n kubevirt scale deploy virt-operator --replicas=1 + kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch \ + '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}},"infra":{"replicas":1}}}' $(call kubectl_wait_exist_and_ready,kubevirt,deployment,virt-operator) $(call kubectl_wait_exist_and_ready,kubevirt,deployment,virt-api) $(call kubectl_wait_exist_and_ready,kubevirt,deployment,virt-controller) $(call kubectl_wait_exist_and_ready,kubevirt,daemonset,virt-handler) - kubectl apply -f "$(KUBEVIRT_TEST_YAML)" - kubectl patch vm testvm --type=merge --patch '{"spec":{"running":true}}' - kubectl wait vm testvm --for=condition=Ready --timeout=2m - .PHONY: kind-install-lb-svc kind-install-lb-svc: $(call kind_load_image,kube-ovn,$(VPC_NAT_GW_IMG)) diff --git a/go.mod b/go.mod index cbf5b58865f..22155a0df9d 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( k8s.io/pod-security-admission v0.29.3 k8s.io/sample-controller v0.29.3 k8s.io/utils v0.0.0-20240310230437-4693a0247e57 + kubevirt.io/api v1.1.1 kubevirt.io/client-go v1.1.1 sigs.k8s.io/controller-runtime v0.17.3 ) @@ -115,8 +116,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/go-kit/kit v0.13.0 // indirect - github.com/go-kit/log v0.2.1 // indirect + github.com/go-kit/kit v0.10.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -127,7 +127,6 @@ require ( github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/glog v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect @@ -272,7 +271,6 @@ require ( k8s.io/kubelet v0.29.3 // indirect k8s.io/legacy-cloud-providers v0.0.0 // indirect k8s.io/mount-utils v0.0.0 // indirect - kubevirt.io/api v1.1.1 // indirect kubevirt.io/containerized-data-importer-api v1.58.1 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect @@ -317,5 +315,5 @@ replace ( k8s.io/mount-utils => k8s.io/mount-utils v0.29.3 k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.3 k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.3 - kubevirt.io/client-go => github.com/kubeovn/kubevirt-client-go v0.0.0-20240218040151-07ffb2d68c98 + kubevirt.io/client-go => github.com/kubeovn/kubevirt-client-go v0.0.0-20240412101643-286960f75418 ) diff --git a/go.sum b/go.sum index 79fdf541556..92dca7c4f4d 100644 --- a/go.sum +++ b/go.sum @@ -1147,12 +1147,10 @@ github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= -github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= @@ -1262,6 +1260,7 @@ github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhO github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -1316,8 +1315,6 @@ github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgR github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= -github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= -github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -1660,8 +1657,8 @@ github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca h1:fTMjoho2et9 github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca/go.mod h1:jY1XeGzkx8ASNJ+SqQSxTESNXARkjvt+I6IJOTnzIjw= github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589 h1:y9exo1hjCsq7jsGUzt11kxhTiEGrGSQ0ZqibAiZk2PQ= github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589/go.mod h1:49upX+/hUyppWIqu58cumojyIwXdkA8k6reA/mQlKuI= -github.com/kubeovn/kubevirt-client-go v0.0.0-20240218040151-07ffb2d68c98 h1:dep4cdkNcs/vcwtVxWjdtkR8VDldyjwwDlCfR/mJWkQ= -github.com/kubeovn/kubevirt-client-go v0.0.0-20240218040151-07ffb2d68c98/go.mod h1:JPGTNzDyfVd42ABNw1lTkFEQOlxTxEAkTKa8RW7Xmp0= +github.com/kubeovn/kubevirt-client-go v0.0.0-20240412101643-286960f75418 h1:eBEuGzPw1L72Sv8xhcUMKw62JahyJV2Ag+OUgqelzu8= +github.com/kubeovn/kubevirt-client-go v0.0.0-20240412101643-286960f75418/go.mod h1:esE8jyUJZGyG7S9wm7tcma1h2Rx48C6at68mHhizWnQ= github.com/kubeovn/libovsdb v0.0.0-20240218023647-f0bc3ce57fcd h1:GhgvSBFKEkVNgDq8IslC04NVuoznreZH/Imz/cr6bhs= github.com/kubeovn/libovsdb v0.0.0-20240218023647-f0bc3ce57fcd/go.mod h1:pTnlGt1JZrncr6pJn/Fhnp3FFTMQRaTVxiSKBLVGa5s= github.com/kubeovn/netlink v0.0.0-20240218024530-d3ada5dae96f h1:3hH6U+CRilak3SxAX9YykAXxxAY25GTEJANLlJNE2jU= diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index dfd325804da..cbc6b0b3d8d 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -680,7 +680,7 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true" if vmKey != "" { - pod.Annotations[fmt.Sprintf(util.VMTemplate, podNet.ProviderName)] = vmName + pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, podNet.ProviderName)] = vmName if err := c.changeVMSubnet(vmName, namespace, podNet.ProviderName, subnet.Name); err != nil { klog.Errorf("vm %s change subnet to %s failed: %v", vmKey, subnet.Name, err) return nil, err diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go index 06315805fc0..edbf667a7da 100644 --- a/pkg/daemon/controller_linux.go +++ b/pkg/daemon/controller_linux.go @@ -530,8 +530,8 @@ func (c *Controller) handlePod(key string) error { } podName := pod.Name - if pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)] != "" { - podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)] + if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] != "" { + podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] } // set default nic bandwidth @@ -561,8 +561,8 @@ func (c *Controller) handlePod(key string) error { } for _, multiNet := range attachNets { provider := fmt.Sprintf("%s.%s.%s", multiNet.Name, multiNet.Namespace, util.OvnProvider) - if pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)] != "" { - podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)] + if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)] != "" { + podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)] } if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" { ifaceID = ovs.PodNameToPortName(podName, pod.Namespace, provider) diff --git a/pkg/daemon/controller_windows.go b/pkg/daemon/controller_windows.go index 7e245529b0b..61d189fd686 100644 --- a/pkg/daemon/controller_windows.go +++ b/pkg/daemon/controller_windows.go @@ -159,8 +159,8 @@ func (c *Controller) handlePod(key string) error { } podName := pod.Name - if pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)] != "" { - podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)] + if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] != "" { + podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] } // set default nic bandwidth @@ -184,8 +184,8 @@ func (c *Controller) handlePod(key string) error { } for _, multiNet := range attachNets { provider := fmt.Sprintf("%s.%s.%s", multiNet.Name, multiNet.Namespace, util.OvnProvider) - if pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)] != "" { - podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)] + if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)] != "" { + podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)] } if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" { ifaceID = ovs.PodNameToPortName(podName, pod.Namespace, provider) diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go index dbbc7705c8a..4ec38c48fca 100644 --- a/pkg/daemon/handler.go +++ b/pkg/daemon/handler.go @@ -129,7 +129,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon loss = pod.Annotations[fmt.Sprintf(util.NetemQosLossAnnotationTemplate, podRequest.Provider)] jitter = pod.Annotations[fmt.Sprintf(util.NetemQosJitterAnnotationTemplate, podRequest.Provider)] providerNetwork = pod.Annotations[fmt.Sprintf(util.ProviderNetworkTemplate, podRequest.Provider)] - vmName = pod.Annotations[fmt.Sprintf(util.VMTemplate, podRequest.Provider)] + vmName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, podRequest.Provider)] ipAddr, err = util.GetIPAddrWithMask(ip, cidr) if err != nil { errMsg := fmt.Errorf("failed to get ip address with mask, %v", err) @@ -486,7 +486,7 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon default: nicType = pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podRequest.Provider)] } - vmName := pod.Annotations[fmt.Sprintf(util.VMTemplate, podRequest.Provider)] + vmName := pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, podRequest.Provider)] if vmName != "" { podRequest.PodName = vmName } diff --git a/pkg/util/const.go b/pkg/util/const.go index 98d371bfefa..bfbed679330 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -21,6 +21,7 @@ const ( VipAnnotation = "ovn.kubernetes.io/vip" AAPsAnnotation = "ovn.kubernetes.io/aaps" ChassisAnnotation = "ovn.kubernetes.io/chassis" + VMAnnotation = "ovn.kubernetes.io/virtualmachine" ExternalIPAnnotation = "ovn.kubernetes.io/external_ip" ExternalMacAnnotation = "ovn.kubernetes.io/external_mac" @@ -85,7 +86,7 @@ const ( ProviderNetworkMtuTemplate = "%s.provider-network.kubernetes.io/mtu" MirrorControlAnnotationTemplate = "%s.kubernetes.io/mirror" PodNicAnnotationTemplate = "%s.kubernetes.io/pod_nic_type" - VMTemplate = "%s.kubernetes.io/virtualmachine" + VMAnnotationTemplate = "%s.kubernetes.io/virtualmachine" ExcludeIpsAnnotation = "ovn.kubernetes.io/exclude_ips" diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 0b4b1b99f92..8d9beccd803 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -6,11 +6,12 @@ import ( "strings" "time" - attachnetclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" + nad "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/clientcmd" "k8s.io/kubernetes/test/e2e/framework" admissionapi "k8s.io/pod-security-admission/api" + "kubevirt.io/client-go/kubecli" "github.com/onsi/ginkgo/v2" @@ -34,8 +35,9 @@ const ( type Framework struct { KubeContext string *framework.Framework - KubeOVNClientSet kubeovncs.Interface - AttachNetClient attachnetclientset.Interface + KubeOVNClientSet kubeovncs.Interface + KubeVirtClientSet kubecli.KubevirtClient + AttachNetClient nad.Interface // master/release-1.10/... ClusterVersion string // 999.999 for master @@ -159,14 +161,25 @@ func (f *Framework) BeforeEach() { ExpectNoError(err) } + if f.KubeVirtClientSet == nil { + ginkgo.By("Creating a KubeVirt client") + config, err := framework.LoadConfig() + ExpectNoError(err) + + config.QPS = f.Options.ClientQPS + config.Burst = f.Options.ClientBurst + f.KubeVirtClientSet, err = kubecli.GetKubevirtClientFromRESTConfig(config) + ExpectNoError(err) + } + if f.AttachNetClient == nil { - ginkgo.By("Creating a nad client") + ginkgo.By("Creating a network attachment definition client") config, err := framework.LoadConfig() ExpectNoError(err) config.QPS = f.Options.ClientQPS config.Burst = f.Options.ClientBurst - f.AttachNetClient, err = attachnetclientset.NewForConfig(config) + f.AttachNetClient, err = nad.NewForConfig(config) ExpectNoError(err) } diff --git a/test/e2e/framework/network-attachment-definition.go b/test/e2e/framework/network-attachment-definition.go index 9ca4c3f8280..2915c7d6463 100644 --- a/test/e2e/framework/network-attachment-definition.go +++ b/test/e2e/framework/network-attachment-definition.go @@ -15,7 +15,11 @@ type NetworkAttachmentDefinitionClient struct { v1.NetworkAttachmentDefinitionInterface } -func (f *Framework) NetworkAttachmentDefinitionClient(namespace string) *NetworkAttachmentDefinitionClient { +func (f *Framework) NetworkAttachmentDefinitionClient() *NetworkAttachmentDefinitionClient { + return f.NetworkAttachmentDefinitionClientNS(f.Namespace.Name) +} + +func (f *Framework) NetworkAttachmentDefinitionClientNS(namespace string) *NetworkAttachmentDefinitionClient { return &NetworkAttachmentDefinitionClient{ f: f, NetworkAttachmentDefinitionInterface: f.AttachNetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace), diff --git a/test/e2e/framework/virtual-machine.go b/test/e2e/framework/virtual-machine.go new file mode 100644 index 00000000000..aff0aec08ca --- /dev/null +++ b/test/e2e/framework/virtual-machine.go @@ -0,0 +1,246 @@ +package framework + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + v1 "kubevirt.io/api/core/v1" + "kubevirt.io/client-go/kubecli" + + "github.com/onsi/gomega" + "github.com/onsi/gomega/format" +) + +// VMClient represents a KubeVirt VM client +type VMClient struct { + f *Framework + kubecli.VirtualMachineInterface +} + +func (f *Framework) VMClient() *VMClient { + return f.VMClientNS(f.Namespace.Name) +} + +func (f *Framework) VMClientNS(namespace string) *VMClient { + return &VMClient{ + f: f, + VirtualMachineInterface: f.KubeVirtClientSet.VirtualMachine(namespace), + } +} + +func (c *VMClient) Get(name string) *v1.VirtualMachine { + vm, err := c.VirtualMachineInterface.Get(context.TODO(), name, &metav1.GetOptions{}) + ExpectNoError(err) + return vm +} + +// Create creates a new vm according to the framework specifications +func (c *VMClient) Create(vm *v1.VirtualMachine) *v1.VirtualMachine { + v, err := c.VirtualMachineInterface.Create(context.TODO(), vm) + ExpectNoError(err, "failed to create vm %s", v.Name) + return c.Get(v.Name) +} + +// CreateSync creates a new vm according to the framework specifications, and waits for it to be ready. +func (c *VMClient) CreateSync(vm *v1.VirtualMachine) *v1.VirtualMachine { + v := c.Create(vm) + ExpectNoError(c.WaitToBeReady(v.Name, timeout)) + // Get the newest vm after it becomes ready + return c.Get(v.Name).DeepCopy() +} + +// Start starts the vm. +func (c *VMClient) Start(name string) *v1.VirtualMachine { + vm := c.Get(name) + if vm.Spec.Running != nil && *vm.Spec.Running { + Logf("vm %s has already been started", name) + return vm + } + + running := true + vm.Spec.Running = &running + _, err := c.VirtualMachineInterface.Update(context.TODO(), vm) + ExpectNoError(err, "failed to update vm %s", name) + return c.Get(name) +} + +// StartSync stops the vm and waits for it to be ready. +func (c *VMClient) StartSync(name string) *v1.VirtualMachine { + _ = c.Start(name) + ExpectNoError(c.WaitToBeReady(name, 2*time.Minute)) + return c.Get(name) +} + +// Stop stops the vm. +func (c *VMClient) Stop(name string) *v1.VirtualMachine { + vm := c.Get(name) + if vm.Spec.Running != nil && !*vm.Spec.Running { + Logf("vm %s has already been stopped", name) + return vm + } + + running := false + vm.Spec.Running = &running + _, err := c.VirtualMachineInterface.Update(context.TODO(), vm) + ExpectNoError(err, "failed to update vm %s", name) + return c.Get(name) +} + +// StopSync stops the vm and waits for it to be stopped. +func (c *VMClient) StopSync(name string) *v1.VirtualMachine { + _ = c.Stop(name) + ExpectNoError(c.WaitToBeStopped(name, 2*time.Minute)) + return c.Get(name) +} + +// Delete deletes a vm if the vm exists +func (c *VMClient) Delete(name string) { + err := c.VirtualMachineInterface.Delete(context.TODO(), name, &metav1.DeleteOptions{}) + ExpectNoError(err, "failed to delete vm %s", name) +} + +// DeleteSync deletes the vm and waits for the vm to disappear for `timeout`. +// If the vm doesn't disappear before the timeout, it will fail the test. +func (c *VMClient) DeleteSync(name string) { + c.Delete(name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for vm %q to disappear", name) +} + +// WaitToDisappear waits the given timeout duration for the specified vm to be ready. +func (c *VMClient) WaitToBeReady(name string, timeout time.Duration) error { + err := k8sframework.Gomega().Eventually(context.TODO(), k8sframework.RetryNotFound(func(ctx context.Context) (*v1.VirtualMachine, error) { + return c.VirtualMachineInterface.Get(ctx, name, &metav1.GetOptions{}) + })).WithTimeout(timeout).Should( + k8sframework.MakeMatcher(func(vm *v1.VirtualMachine) (func() string, error) { + if vm.Status.Ready { + return nil, nil + } + return func() string { + return fmt.Sprintf("expected vm status to be ready, got status instead:\n%s", format.Object(vm.Status, 1)) + }, nil + })) + if err != nil { + return fmt.Errorf("expected vm %s to be ready: %w", name, err) + } + return nil +} + +// WaitToDisappear waits the given timeout duration for the specified vm to be stopped. +func (c *VMClient) WaitToBeStopped(name string, timeout time.Duration) error { + err := k8sframework.Gomega().Eventually(context.TODO(), k8sframework.RetryNotFound(func(ctx context.Context) (*v1.VirtualMachine, error) { + return c.VirtualMachineInterface.Get(ctx, name, &metav1.GetOptions{}) + })).WithTimeout(timeout).Should( + k8sframework.MakeMatcher(func(vm *v1.VirtualMachine) (func() string, error) { + if !vm.Status.Created { + return nil, nil + } + return func() string { + return fmt.Sprintf("expected vm status to be stopped, got status instead:\n%s", format.Object(vm.Status, 1)) + }, nil + })) + if err != nil { + return fmt.Errorf("expected vm %s to be stopped: %w", name, err) + } + return nil +} + +// WaitToDisappear waits the given timeout duration for the specified vm to disappear. +func (c *VMClient) WaitToDisappear(name string, _, timeout time.Duration) error { + err := k8sframework.Gomega().Eventually(context.Background(), k8sframework.HandleRetry(func(ctx context.Context) (*v1.VirtualMachine, error) { + vm, err := c.VirtualMachineInterface.Get(ctx, name, &metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + return vm, err + })).WithTimeout(timeout).Should(gomega.BeNil()) + if err != nil { + return fmt.Errorf("expected vm %s to not be found: %w", name, err) + } + return nil +} + +func MakeVM(name, image, size string, running bool) *v1.VirtualMachine { + vm := &v1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.VirtualMachineSpec{ + Running: &running, + Template: &v1.VirtualMachineInstanceTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "kubevirt.io/size": size, + "kubevirt.io/domain": name, + }, + }, + Spec: v1.VirtualMachineInstanceSpec{ + Domain: v1.DomainSpec{ + Devices: v1.Devices{ + Disks: []v1.Disk{ + { + Name: "containerdisk", + DiskDevice: v1.DiskDevice{ + Disk: &v1.DiskTarget{ + Bus: v1.DiskBusVirtio, + }, + }, + }, + { + Name: "cloudinitdisk", + DiskDevice: v1.DiskDevice{ + Disk: &v1.DiskTarget{ + Bus: v1.DiskBusVirtio, + }, + }, + }, + }, + Interfaces: []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.DefaultMasqueradeNetworkInterface().InterfaceBindingMethod, + }, + }, + }, + Resources: v1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("64M"), + }, + }, + }, + Networks: []v1.Network{ + { + Name: "default", + NetworkSource: v1.DefaultPodNetwork().NetworkSource, + }, + }, + Volumes: []v1.Volume{ + { + Name: "containerdisk", + VolumeSource: v1.VolumeSource{ + ContainerDisk: &v1.ContainerDiskSource{ + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + }, + { + Name: "cloudinitdisk", + VolumeSource: v1.VolumeSource{ + CloudInitNoCloud: &v1.CloudInitNoCloudSource{ + UserDataBase64: "SGkuXG4=", + }, + }, + }, + }, + }, + }, + }, + } + return vm +} diff --git a/test/e2e/iptables-vpc-nat-gw/e2e_test.go b/test/e2e/iptables-vpc-nat-gw/e2e_test.go index d3f01431ebb..d51c6c9173d 100644 --- a/test/e2e/iptables-vpc-nat-gw/e2e_test.go +++ b/test/e2e/iptables-vpc-nat-gw/e2e_test.go @@ -240,7 +240,7 @@ var _ = framework.Describe("[group:iptables-vpc-nat-gw]", func() { ginkgo.BeforeEach(func() { containerID = "" cs = f.ClientSet - attachNetClient = f.NetworkAttachmentDefinitionClient(framework.KubeOvnNamespace) + attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) subnetClient = f.SubnetClient() vpcClient = f.VpcClient() vpcNatGwClient = f.VpcNatGatewayClient() @@ -1085,7 +1085,7 @@ var _ = framework.Describe("[group:qos-policy]", func() { containerID = "" cs = f.ClientSet podClient = f.PodClient() - attachNetClient = f.NetworkAttachmentDefinitionClient(framework.KubeOvnNamespace) + attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) subnetClient = f.SubnetClient() vpcClient = f.VpcClient() vpcNatGwClient = f.VpcNatGatewayClient() diff --git a/test/e2e/kubevirt/e2e_test.go b/test/e2e/kubevirt/e2e_test.go index 895c0050303..1881a149027 100644 --- a/test/e2e/kubevirt/e2e_test.go +++ b/test/e2e/kubevirt/e2e_test.go @@ -3,13 +3,16 @@ package kubevirt import ( "context" "flag" + "fmt" "testing" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/kubernetes/test/e2e" k8sframework "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/config" + v1 "kubevirt.io/api/core/v1" "github.com/onsi/ginkgo/v2" @@ -17,6 +20,8 @@ import ( "github.com/kubeovn/kube-ovn/test/e2e/framework" ) +const image = "quay.io/kubevirt/cirros-container-disk-demo:latest" + func init() { klog.SetOutput(ginkgo.GinkgoWriter) @@ -34,17 +39,31 @@ func TestE2E(t *testing.T) { var _ = framework.Describe("[group:kubevirt]", func() { f := framework.NewDefaultFramework("kubevirt") + var vmName, namespaceName string var podClient *framework.PodClient + var vmClient *framework.VMClient ginkgo.BeforeEach(func() { - podClient = f.PodClientNS("default") - }) - - framework.ConformanceIt("Kubevirt vm pod should keep ip", func() { f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12.") - ginkgo.By("Get kubevirt vm pod") + namespaceName = f.Namespace.Name + vmName = "vm-" + framework.RandomSuffix() + podClient = f.PodClientNS(namespaceName) + vmClient = f.VMClientNS(namespaceName) + + ginkgo.By("Creating vm " + vmName) + vm := framework.MakeVM(vmName, image, "small", true) + _ = vmClient.CreateSync(vm) + }) + ginkgo.AfterEach(func() { + ginkgo.By("Deleting vm " + vmName) + vmClient.DeleteSync(vmName) + }) + + framework.ConformanceIt("should be able to keep pod ips after vm pod is deleted", func() { + ginkgo.By("Getting pod of vm " + vmName) + labelSelector := fmt.Sprintf("%s=%s", v1.VirtualMachineNameLabel, vmName) podList, err := podClient.List(context.TODO(), metav1.ListOptions{ - LabelSelector: "vm.kubevirt.io/name=testvm", + LabelSelector: labelSelector, }) framework.ExpectNoError(err) framework.ExpectHaveLen(podList.Items, 1) @@ -53,32 +72,72 @@ var _ = framework.Describe("[group:kubevirt]", func() { pod := &podList.Items[0] framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") - framework.ExpectHaveKeyWithValue(pod.Annotations, "ovn.kubernetes.io/virtualmachine", "testvm") - ipAddr := pod.Annotations[util.IPAddressAnnotation] + framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName) + ips := pod.Status.PodIPs ginkgo.By("Deleting pod " + pod.Name) podClient.DeleteSync(pod.Name) framework.ExpectNoError(err) - ginkgo.By("Check kubevirt vm pod after rebuild") + ginkgo.By("Waiting for vm " + vmName + " to be ready") + err = vmClient.WaitToBeReady(vmName, 2*time.Minute) + framework.ExpectNoError(err) + + ginkgo.By("Getting pod of vm " + vmName) podList, err = podClient.List(context.TODO(), metav1.ListOptions{ - LabelSelector: "vm.kubevirt.io/name=testvm", + LabelSelector: labelSelector, }) framework.ExpectNoError(err) framework.ExpectHaveLen(podList.Items, 1) + ginkgo.By("Validating new pod annotations") pod = &podList.Items[0] - ginkgo.By("Waiting for pod " + pod.Name + " to be running") - podClient.WaitForRunning(pod.Name) + framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName) + + ginkgo.By("Checking whether pod ips are changed") + framework.ExpectConsistOf(ips, pod.Status.PodIPs) + }) + + framework.ConformanceIt("should be able to keep pod ips after the vm is restarted", func() { + ginkgo.By("Getting pod of vm " + vmName) + labelSelector := fmt.Sprintf("%s=%s", v1.VirtualMachineNameLabel, vmName) + podList, err := podClient.List(context.TODO(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + framework.ExpectNoError(err) + framework.ExpectHaveLen(podList.Items, 1) + + ginkgo.By("Validating pod annotations") + pod := &podList.Items[0] + framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName) + ips := pod.Status.PodIPs + + ginkgo.By("Stopping vm " + vmName) + vmClient.StopSync(vmName) + framework.ExpectNoError(err) + + ginkgo.By("Starting vm " + vmName) + vmClient.StartSync(vmName) + framework.ExpectNoError(err) + + ginkgo.By("Getting pod of vm " + vmName) + podList, err = podClient.List(context.TODO(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + framework.ExpectNoError(err) + framework.ExpectHaveLen(podList.Items, 1) ginkgo.By("Validating new pod annotations") - pod = podClient.GetPod(pod.Name) + pod = &podList.Items[0] framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") - framework.ExpectHaveKeyWithValue(pod.Annotations, "ovn.kubernetes.io/virtualmachine", "testvm") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName) - ginkgo.By("Check vm pod ip unchanged" + pod.Name) - ipNewAddr := pod.Annotations[util.IPAddressAnnotation] - framework.ExpectEqual(ipAddr, ipNewAddr) + ginkgo.By("Checking whether pod ips are changed") + framework.ExpectConsistOf(ips, pod.Status.PodIPs) }) }) diff --git a/test/e2e/multus/e2e_test.go b/test/e2e/multus/e2e_test.go index f3009a67821..ec4cdc73f12 100644 --- a/test/e2e/multus/e2e_test.go +++ b/test/e2e/multus/e2e_test.go @@ -70,7 +70,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { cidr = framework.RandomCIDR(f.ClusterIPFamily) podClient = f.PodClient() subnetClient = f.SubnetClient() - nadClient = f.NetworkAttachmentDefinitionClient(namespaceName) + nadClient = f.NetworkAttachmentDefinitionClientNS(namespaceName) if image == "" { image = framework.GetKubeOvnImage(f.ClientSet)