e2e.yml

# Copyright 2023 The Kubernetes Authors.
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: e2e

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - "**"

env:
  K8S_VERSION: "v1.27.3"
  KIND_VERSION: "v0.20.0"
  KIND_CLUSTER_NAME: "nodeipam"
  REGISTRY: gcr.io
  IMAGE_NAME: k8s-staging-networking/node-ipam-controller

jobs:
  build-image:
    name: build-image
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: Install Go
      uses: actions/setup-go@v4
      with:
        go-version: 1.21.x

    - name: Build docker
      run: |
        GIT_TAG=ci-e2e-test PUSH=--load make image-build
        mkdir _output
        docker images
        echo "image:" $REGISTRY/$IMAGE_NAME:ci-e2e-test
        docker save --output _output/node-ipam-controller-image.tar $REGISTRY/$IMAGE_NAME:ci-e2e-test

    - uses: actions/upload-artifact@v4
      with:
        name: test-image
        path: _output/node-ipam-controller-image.tar

  e2e:
    name: e2e
    runs-on: ubuntu-latest
    timeout-minutes: 100
    strategy:
      fail-fast: false
      matrix:
        ipFamily: ["ipv4", "ipv6", "dual"]
    env:
      JOB_NAME: "nodeipam-e2e-${{ matrix.ipFamily }}"
      IP_FAMILY: ${{ matrix.ipFamily }}
    needs:
      - build-image
    steps:
    - name: Check out code
      uses: actions/checkout@v4

    - name: Enable ipv4 and ipv6 forwarding
      run: |
        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        sudo sysctl -w net.ipv4.ip_forward=1

    - name: Set up environment
      run: ./hack/github/setup-test-env.sh ${{ env.K8S_VERSION }} ${{ env.KIND_VERSION }}

    - name: Create multi node cluster
      run: |
        # output_dir
        mkdir -p _artifacts
        # create cluster
        cat <<EOF | /usr/local/bin/kind create cluster \
          --name ${{ env.KIND_CLUSTER_NAME}}           \
          --image kindest/node:${{ env.K8S_VERSION }}  \
          -v7 --wait 1m --retain --config=-
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        networking:
          ipFamily: ${IP_FAMILY}
        nodes:
        - role: control-plane
          kubeadmConfigPatches:
          - |
            kind: ClusterConfiguration
            controllerManager:
              extraArgs:
                allocate-node-cidrs: "false"
        - role: worker
        - role: worker
        EOF
        /usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf

    - uses: actions/download-artifact@v4.1.7
      with:
        name: test-image

    - name: Install node-ipam-controller
      run: |
        # preload image
        docker load --input node-ipam-controller-image.tar
        /usr/local/bin/kind load docker-image $REGISTRY/$IMAGE_NAME:ci-e2e-test --name ${{ env.KIND_CLUSTER_NAME}}

        # install CRD separately
        /usr/local/bin/kubectl create -f charts/node-ipam-controller/gen/crds/networking.x-k8s.io_clustercidrs.yaml
        # create ClusterCIDR
        /usr/local/bin/kubectl create -f examples/clustercidr-${IP_FAMILY}.yaml
        # configure controller version and disable installing CRD
        sed -i 's/tag.*/tag: "ci-e2e-test"/' charts/node-ipam-controller/values.yaml
        sed -i 's/installCRDs: true/installCRDs: false/' charts/node-ipam-controller/values.yaml
        # install the controller
        helm template charts/node-ipam-controller --values charts/node-ipam-controller/values.yaml --set image.repository=$REGISTRY/$IMAGE_NAME  > temp.yaml
        cat temp.yaml
        /usr/local/bin/kubectl apply -f ./temp.yaml
        /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods -l app.kubernetes.io/name=node-ipam-controller
        /usr/local/bin/kubectl get nodes -o wide
        /usr/local/bin/kubectl get pods -A

    - name: Get Cluster status
      run: |
        # install kindnet because of bug https://github.com/kubernetes-sigs/kind/pull/3438
        kubectl apply -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml
        /usr/local/bin/kubectl get nodes -o wide
        /usr/local/bin/kubectl get pods -A -o wide
        /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns
        /usr/local/bin/kubectl get nodes -o wide
        /usr/local/bin/kubectl get pods -A -o wide

    - name: Workaround CoreDNS for IPv6 airgapped
      if: ${{ matrix.ipFamily == 'ipv6' }}
      run: |
        # Patch CoreDNS to work in Github CI
        # 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
        # to work in an offline environment:
        # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
        # 2. Github CI adds following domains to resolv.conf search field:
        # .net.
        # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
        # otherwise pods stops trying to resolve the domain.
        # Get the current config
        original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
        echo "Original CoreDNS config:"
        echo "${original_coredns}"
        # Patch it
        fixed_coredns=$(
          printf '%s' "${original_coredns}" | sed \
            -e 's/^.*kubernetes cluster\.local/& net/' \
            -e '/^.*upstream$/d' \
            -e '/^.*fallthrough.*$/d' \
            -e '/^.*forward . \/etc\/resolv.conf$/d' \
            -e '/^.*loop$/d' \
        )
        echo "Patched CoreDNS config:"
        echo "${fixed_coredns}"
        printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -

    - name: Run tests
      run: |
        export KUBERNETES_CONFORMANCE_TEST='y'
        export E2E_REPORT_DIR=${PWD}/_artifacts

        # Run tests
        /usr/local/bin/ginkgo --nodes=25                  \
          --focus="Networking Granular Checks"            \
          --skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance"   \
          /usr/local/bin/e2e.test                         \
          --                                              \
          --kubeconfig=${PWD}/_artifacts/kubeconfig.conf  \
          --provider=local                                \
          --dump-logs-on-failure=false                    \
          --report-dir=${E2E_REPORT_DIR}                  \
          --disable-log-dump=true

    - name: Upload Junit Reports
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: kind-junit-nodeipam-e2e-${{ github.run_id }}-${{ matrix.ipFamily }}
        path: './_artifacts/*.xml'

    - name: Export logs
      if: always()
      run: |
        /usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs

    - name: Upload logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: kind-logs-nodeipam-e2e-${{ github.run_id }}-${{ matrix.ipFamily }}
        path: ./_artifacts/logs