[Release-1.28] Unit, Integration and Install Testing Overhaul

.github/actions/vagrant-setup/action.yaml

@@ -0,0 +1,33 @@
+name: 'Setup Vagrant and Libvirt'
+description: 'A composite action that installs latest versions of vagrant and libvirt for use on ubuntu based runners'
+runs:
+  using: 'composite'
+  steps:
+    - name: Add vagrant to apt-get sources
+      shell: bash
+      run: |
+        curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+        echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+        sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list
+    - name: Install vagrant and libvirt
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant
+        sudo systemctl enable --now libvirtd
+    - name: Build vagrant dependencies
+      shell: bash
+      run: |
+        sudo apt-get build-dep -y vagrant ruby-libvirt
+        sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
+    # This is a workaround for the libvirt group not being available in the current shell
+    # https://github.com/actions/runner-images/issues/7670#issuecomment-1900711711
+    - name: Make the libvirt socket rw accessible to everyone
+      shell: bash
+      run: |
+        sudo chmod a+rw /var/run/libvirt/libvirt-sock
+
+
+    - name: Install vagrant-libvirt plugin
+      shell: bash
+      run: vagrant plugin install vagrant-libvirt

.github/workflows/integration.yaml

@@ -0,0 +1,121 @@
+name: Integration Test Coverage
+on: 
+  push:
+    paths-ignore:
+      - "**.md"
+      - "channel.yaml"
+      - "install.sh"
+      - "tests/**"
+      - "!tests/integration**"
+      - ".github/**"
+      - "!.github/workflows/integration.yaml"
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - "channel.yaml"
+      - "install.sh"
+      - "tests/**"
+      - "!tests/integration**"
+      - "!tests/e2e**"
+      - ".github/**"
+      - "!.github/workflows/integration.yaml"
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build RKE2 Images and Binary
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+    - name: Find Go Version for Build
+      id: go-finder
+      run: |
+        GOOS=linux GOARCH=amd64 . ./scripts/version.sh
+        set +x
+        VERSION_GOLANG=$(echo $VERSION_GOLANG | sed 's/go//')
+        echo "VERSION_GOLANG=${VERSION_GOLANG}" >> "$GITHUB_OUTPUT"
+    - uses: actions/setup-go@v5
+      with:
+        go-version: ${{ steps.go-finder.outputs.VERSION_GOLANG }}
+        cache-dependency-path: |
+          **/go.sum
+          **/go.mod
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Install OS Packages
+      run: sudo apt-get install -y libarchive-tools
+    - name: Build RKE2 Binary and Runtime Image
+      run: |
+        GOCOVER=true make build-binary
+        make package-image-runtime
+        cp ./bin/rke2 ./build/images/rke2-binary
+    # Can only upload from a single path, so we need to copy the binary to the image directory
+    - name: Upload RKE2 Binary and Runtime Image
+      uses: actions/upload-artifact@v4
+      with:
+        name: rke2-test-artifacts
+        path: ./build/images/*
+  test:
+    needs: build
+    name: Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        itest: [etcdsnapshot]
+      max-parallel: 3
+    env:
+      GOCOVERDIR: /tmp/rke2cov
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: 'go.mod'
+        cache-dependency-path: |
+          **/go.sum
+          **/go.mod
+    - name: Setup Build Directories
+      run: |
+        mkdir -p ./bin ./build/images
+    - name: Download RKE2 Binary and Runtime Image
+      uses: actions/download-artifact@v4
+      with:
+        name: rke2-test-artifacts
+        path: ./build/images
+    - name: Setup Binary
+      run: |
+        mv ./build/images/rke2-binary ./bin/rke2
+        chmod +x ./bin/rke2
+    - name: Run Integration Tests
+      run: | 
+        mkdir -p $GOCOVERDIR
+        sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration
+    - name: Generate coverage report
+      run:  go tool covdata textfmt -i $GOCOVERDIR -o ${{ matrix.itest }}.out
+    - name: Upload Results To Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./${{ matrix.itest }}.out
+        flags: inttests # optional
+        verbose: true # optional (default = false)
+    - name: On Failure, Dump Server Logs
+      if: ${{ failure() }}
+      run: cat ./tests/integration/${{ matrix.itest }}/r2log.txt
+    - name: On Failure, Launch Debug Session
+      uses: dereknola/action-upterm@main
+      if: ${{ failure() }}
+      with:
+        ## If no one connects after 5 minutes, shut down server.
+        wait-timeout-minutes: 5
+        limit-access-to-actor: true

.github/workflows/nightly-install.yaml

@@ -1,22 +1,19 @@
 name: Nightly Install
 on:
-  schedule:
-    - cron: "0 0 * * 1-5"
   workflow_dispatch: {}
 jobs:
   test:
     name: "Smoke Test"
-    # nested virtualization is only available on macOS hosts
-    runs-on: macos-12
+    runs-on: ubuntu-latest
     timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
         channel: [stable]
-        vm: [centos-7, rocky-8, opensuse-microos, opensuse-leap, ubuntu-focal, windows-2019, windows-2022]
+        vm: [centos-7, rocky-8, opensuse-leap, ubuntu-2004, windows-2019, windows-2022]
         include:
           - {channel: latest, vm: rocky-8}
-          - {channel: latest, vm: ubuntu-focal}
+          - {channel: latest, vm: ubuntu-2004}
       max-parallel: 2
     defaults:
       run:
@@ -27,22 +24,25 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v4
         with: {fetch-depth: 1}
+      # Don't cache Windows VMs, they are 5GB each, which would eat our entire 10GB cache
       - name: "Vagrant Cache"
-        uses: actions/cache@v3
+        if: ${{ !contains(matrix.vm, 'windows') }}
+        uses: actions/cache@v4
         with:
           path: |
-            ~/.vagrant.d/boxes
-            ~/.vagrant.d/gems
-          key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }}
+             ~/.vagrant.d/boxes
+          key: vagrant-box-${{ matrix.vm }}
         id: vagrant-cache
         continue-on-error: true
+      - name: Set up vagrant and libvirt
+        uses: ./.github/actions/vagrant-setup
       - name: "Vagrant Plugin(s)"
         run: vagrant plugin install vagrant-reload vagrant-rke2
       - name: "Vagrant Up ⏩ Install RKE2"
         run: vagrant up
       - name: "⏳ Node"
         if: ${{ !contains(matrix.vm, 'windows') }}
-        run: vagrant provision --provision-with=rke2-wait-for-node
+        run: vagrant provision --provision-with=rke2-wait-for-cp
       - name: "⏳ Canal"
         if: ${{ !contains(matrix.vm, 'windows') }}
         run: vagrant provision --provision-with=rke2-wait-for-canal

.github/workflows/unittest.yaml

@@ -0,0 +1,47 @@
+name: Unit Test Coverage
+on:
+  push:
+    paths-ignore:
+      - "**.md"
+      - "channels.yaml"
+      - "install.sh"
+      - "tests/**"
+      - ".github/**"
+      - "!.github/workflows/unittest.yaml"
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - "channels.yaml"
+      - "install.sh"
+      - "tests/**"
+      - ".github/**"
+      - "!.github/workflows/unittest.yaml"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with: 
+        go-version-file: 'go.mod'
+    - name: Run Unit Tests
+      run: | 
+        go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Unit
+        go tool cover -func coverage.out
+    - name: Upload Results To Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./coverage.out
+        flags: unittests # optional
+        verbose: true # optional (default = false)

Dockerfile

@@ -30,7 +30,7 @@ RUN zypper install -y systemd-rpm-macros
 
 # Dapper/Drone/CI environment
 FROM build AS dapper
-ENV DAPPER_ENV GODEBUG REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY
+ENV DAPPER_ENV GODEBUG GOCOVER REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
 ENV DAPPER_OUTPUT ./dist ./bin ./build

Makefile

@@ -47,18 +47,10 @@ build-images:                             ## Build all images and image tarballs
 build-windows-images:                     ## Build only the Windows images and tarballs (including airgap)
 	./scripts/build-windows-images
 
-.PHONY: build-image-kubernetes
-build-image-kubernetes:                   ## Build the kubernetes image
-	./scripts/build-image-kubernetes
-
 .PHONY: build-image-runtime
 build-image-runtime:                      ## Build the runtime image
 	./scripts/build-image-runtime
 
-.PHONY: publish-image-kubernetes
-publish-image-kubernetes: build-image-kubernetes
-	./scripts/publish-image-kubernetes
-
 .PHONY: publish-image-runtime
 publish-image-runtime: build-image-runtime
 	./scripts/publish-image-runtime
@@ -136,6 +128,10 @@ package-images: build-images		## Package docker images for airgap environment
 package-windows-images: build-windows-images		## Package Windows crane images for airgap environment
 	./scripts/package-windows-images
 
+.PHONY: package-image-runtime
+package-image-runtime: build-image-runtime		## Package runtime image for GH Actions testing
+	./scripts/package-image-runtime
+
 .PHONY: package-bundle
 package-bundle: build-binary					## Package the tarball bundle
 	./scripts/package-bundle

pkg/rke2/rke2_linux_test.go

@@ -28,7 +28,6 @@ func Test_UnitInitExecutor(t *testing.T) {
 		{
 			name: "agent",
 			args: args{
-				clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil),
 				cfg: Config{
 					ControlPlaneProbeConf:        []string{"kube-proxy-startup-initial-delay-seconds=42"},
 					ControlPlaneResourceLimits:   []string{"kube-proxy-cpu=123m"},
@@ -62,7 +61,6 @@ func Test_UnitInitExecutor(t *testing.T) {
 		{
 			name: "server",
 			args: args{
-				clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil),
 				cfg: Config{
 					ControlPlaneProbeConf:        []string{"kube-proxy-startup-initial-delay-seconds=123"},
 					ControlPlaneResourceLimits:   []string{"kube-proxy-cpu=42m"},
@@ -96,7 +94,6 @@ func Test_UnitInitExecutor(t *testing.T) {
 		{
 			name: "bad probe conf",
 			args: args{
-				clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil),
 				cfg: Config{
 					ControlPlaneProbeConf: []string{"kube-proxy-startup-initial-delay-seconds=-123"},
 				},
@@ -106,7 +103,6 @@ func Test_UnitInitExecutor(t *testing.T) {
 		{
 			name: "bad control plane limits",
 			args: args{
-				clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil),
 				cfg: Config{
 					ControlPlaneResourceLimits: []string{"kube-proxy-cpu"},
 				},
@@ -116,7 +112,6 @@ func Test_UnitInitExecutor(t *testing.T) {
 		{
 			name: "bad control plane requests",
 			args: args{
-				clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil),
 				cfg: Config{
 					ControlPlaneResourceRequests: []string{"kube-proxy-memory"},
 				},
@@ -126,6 +121,10 @@ func Test_UnitInitExecutor(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			// Override the pss location so we attempt to create a file that needs sudo, not what we are testing anyways
+			flagSet := flag.NewFlagSet("test", 0)
+			flagSet.String("pod-security-admission-config-file", "/tmp/pss.yaml", "")
+			tt.args.clx = cli.NewContext(nil, flagSet, nil)
 			got, err := initExecutor(tt.args.clx, tt.args.cfg, tt.args.isServer)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("initExecutor() error = %v, wantErr %v", err, tt.wantErr)