diff --git a/.github/actions/vagrant-setup/action.yaml b/.github/actions/vagrant-setup/action.yaml new file mode 100644 index 0000000000..37f268809e --- /dev/null +++ b/.github/actions/vagrant-setup/action.yaml @@ -0,0 +1,33 @@ +name: 'Setup Vagrant and Libvirt' +description: 'A composite action that installs latest versions of vagrant and libvirt for use on ubuntu based runners' +runs: + using: 'composite' + steps: + - name: Add vagrant to apt-get sources + shell: bash + run: | + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list + - name: Install vagrant and libvirt + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant + sudo systemctl enable --now libvirtd + - name: Build vagrant dependencies + shell: bash + run: | + sudo apt-get build-dep -y vagrant ruby-libvirt + sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev + # This is a workaround for the libvirt group not being available in the current shell + # https://github.com/actions/runner-images/issues/7670#issuecomment-1900711711 + - name: Make the libvirt socket rw accessible to everyone + shell: bash + run: | + sudo chmod a+rw /var/run/libvirt/libvirt-sock + + + - name: Install vagrant-libvirt plugin + shell: bash + run: vagrant plugin install vagrant-libvirt \ No newline at end of file diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml new file mode 100644 index 0000000000..b438e6f4c5 --- /dev/null +++ b/.github/workflows/integration.yaml @@ -0,0 +1,121 @@ +name: Integration Test Coverage +on: + push: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/integration**" + - ".github/**" + - "!.github/workflows/integration.yaml" + pull_request: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/integration**" + - "!tests/e2e**" + - ".github/**" + - "!.github/workflows/integration.yaml" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + build: + name: Build RKE2 Images and Binary + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Find Go Version for Build + id: go-finder + run: | + GOOS=linux GOARCH=amd64 . ./scripts/version.sh + set +x + VERSION_GOLANG=$(echo $VERSION_GOLANG | sed 's/go//') + echo "VERSION_GOLANG=${VERSION_GOLANG}" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@v5 + with: + go-version: ${{ steps.go-finder.outputs.VERSION_GOLANG }} + cache-dependency-path: | + **/go.sum + **/go.mod + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install OS Packages + run: sudo apt-get install -y libarchive-tools + - name: Build RKE2 Binary and Runtime Image + run: | + GOCOVER=true make build-binary + make package-image-runtime + cp ./bin/rke2 ./build/images/rke2-binary + # Can only upload from a single path, so we need to copy the binary to the image directory + - name: Upload RKE2 Binary and Runtime Image + uses: actions/upload-artifact@v4 + with: + name: rke2-test-artifacts + path: ./build/images/* + test: + needs: build + name: Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + itest: [etcdsnapshot] + max-parallel: 3 + env: + GOCOVERDIR: /tmp/rke2cov + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache-dependency-path: | + **/go.sum + **/go.mod + - name: Setup Build Directories + run: | + mkdir -p ./bin ./build/images + - name: Download RKE2 Binary and Runtime Image + uses: actions/download-artifact@v4 + with: + name: rke2-test-artifacts + path: ./build/images + - name: Setup Binary + run: | + mv ./build/images/rke2-binary ./bin/rke2 + chmod +x ./bin/rke2 + - name: Run Integration Tests + run: | + mkdir -p $GOCOVERDIR + sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration + - name: Generate coverage report + run: go tool covdata textfmt -i $GOCOVERDIR -o ${{ matrix.itest }}.out + - name: Upload Results To Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./${{ matrix.itest }}.out + flags: inttests # optional + verbose: true # optional (default = false) + - name: On Failure, Dump Server Logs + if: ${{ failure() }} + run: cat ./tests/integration/${{ matrix.itest }}/r2log.txt + - name: On Failure, Launch Debug Session + uses: dereknola/action-upterm@main + if: ${{ failure() }} + with: + ## If no one connects after 5 minutes, shut down server. + wait-timeout-minutes: 5 + limit-access-to-actor: true diff --git a/.github/workflows/nightly-install.yaml b/.github/workflows/nightly-install.yaml index b0aa8add12..cdf043d30d 100644 --- a/.github/workflows/nightly-install.yaml +++ b/.github/workflows/nightly-install.yaml @@ -1,22 +1,19 @@ name: Nightly Install on: - schedule: - - cron: "0 0 * * 1-5" workflow_dispatch: {} jobs: test: name: "Smoke Test" - # nested virtualization is only available on macOS hosts - runs-on: macos-12 + runs-on: ubuntu-latest timeout-minutes: 40 strategy: fail-fast: false matrix: channel: [stable] - vm: [centos-7, rocky-8, opensuse-microos, opensuse-leap, ubuntu-focal, windows-2019, windows-2022] + vm: [centos-7, rocky-8, opensuse-leap, ubuntu-2004, windows-2019, windows-2022] include: - {channel: latest, vm: rocky-8} - - {channel: latest, vm: ubuntu-focal} + - {channel: latest, vm: ubuntu-2004} max-parallel: 2 defaults: run: @@ -27,22 +24,25 @@ jobs: - name: "Checkout" uses: actions/checkout@v3 with: {fetch-depth: 1} + # Don't cache Windows VMs, they are 5GB each, which would eat our entire 10GB cache - name: "Vagrant Cache" - uses: actions/cache@v3 + if: ${{ !contains(matrix.vm, 'windows') }} + uses: actions/cache@v4 with: path: | - ~/.vagrant.d/boxes - ~/.vagrant.d/gems - key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }} + ~/.vagrant.d/boxes + key: vagrant-box-${{ matrix.vm }} id: vagrant-cache continue-on-error: true + - name: Set up vagrant and libvirt + uses: ./.github/actions/vagrant-setup - name: "Vagrant Plugin(s)" run: vagrant plugin install vagrant-reload vagrant-rke2 - name: "Vagrant Up ⏩ Install RKE2" run: vagrant up - name: "⏳ Node" if: ${{ !contains(matrix.vm, 'windows') }} - run: vagrant provision --provision-with=rke2-wait-for-node + run: vagrant provision --provision-with=rke2-wait-for-cp - name: "⏳ Canal" if: ${{ !contains(matrix.vm, 'windows') }} run: vagrant provision --provision-with=rke2-wait-for-canal diff --git a/.github/workflows/unitcoverage.yaml b/.github/workflows/unitcoverage.yaml deleted file mode 100644 index c19cad73f6..0000000000 --- a/.github/workflows/unitcoverage.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# name: Unit Test Coverage -# on: -# push: -# paths-ignore: -# - "install.sh" -# - "tests/vagrant/**" -# pull_request: -# paths-ignore: -# - "install.sh" -# - "tests/vagrant/**" -# jobs: -# test: -# name: Unit Tests -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-20.04] -# timeout-minutes: 20 -# steps: -# - name: Install Go -# uses: actions/setup-go@v3 -# with: -# go-version: '1.19.3' -# - name: Checkout -# uses: actions/checkout@v3 -# with: -# fetch-depth: 1 -# - name: Run Unit Tests -# run: | -# go test -coverpkg=./... -covermode=atomic -coverprofile=coverage.out ./pkg/... -run Unit -# go tool cover -func coverage.out -# - name: On Failure, Launch Debug Session -# if: ${{ failure() }} -# uses: mxschmitt/action-tmate@v3 -# timeout-minutes: 2 -# - name: Upload Results To Codecov -# uses: codecov/codecov-action@v1 -# with: -# files: ./coverage.out -# flags: unittests # optional -# verbose: true # optional (default = false) diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml new file mode 100644 index 0000000000..24a66f54f4 --- /dev/null +++ b/.github/workflows/unittest.yaml @@ -0,0 +1,47 @@ +name: Unit Test Coverage +on: + push: + paths-ignore: + - "**.md" + - "channels.yaml" + - "install.sh" + - "tests/**" + - ".github/**" + - "!.github/workflows/unittest.yaml" + pull_request: + paths-ignore: + - "**.md" + - "channels.yaml" + - "install.sh" + - "tests/**" + - ".github/**" + - "!.github/workflows/unittest.yaml" + +permissions: + contents: read + +jobs: + test: + name: Unit Tests + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: Run Unit Tests + run: | + go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Unit + go tool cover -func coverage.out + - name: Upload Results To Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.out + flags: unittests # optional + verbose: true # optional (default = false) diff --git a/Dockerfile b/Dockerfile index c86098341c..e19ea80684 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN zypper install -y systemd-rpm-macros # Dapper/Drone/CI environment FROM build AS dapper -ENV DAPPER_ENV GODEBUG REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY +ENV DAPPER_ENV GODEBUG GOCOVER REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY ARG DAPPER_HOST_ARCH ENV ARCH $DAPPER_HOST_ARCH ENV DAPPER_OUTPUT ./dist ./bin ./build diff --git a/Makefile b/Makefile index 73444032fa..24dc8396f6 100644 --- a/Makefile +++ b/Makefile @@ -47,18 +47,10 @@ build-images: ## Build all images and image tarballs build-windows-images: ## Build only the Windows images and tarballs (including airgap) ./scripts/build-windows-images -.PHONY: build-image-kubernetes -build-image-kubernetes: ## Build the kubernetes image - ./scripts/build-image-kubernetes - .PHONY: build-image-runtime build-image-runtime: ## Build the runtime image ./scripts/build-image-runtime -.PHONY: publish-image-kubernetes -publish-image-kubernetes: build-image-kubernetes - ./scripts/publish-image-kubernetes - .PHONY: publish-image-runtime publish-image-runtime: build-image-runtime ./scripts/publish-image-runtime @@ -136,6 +128,10 @@ package-images: build-images ## Package docker images for airgap environment package-windows-images: build-windows-images ## Package Windows crane images for airgap environment ./scripts/package-windows-images +.PHONY: package-image-runtime +package-image-runtime: build-image-runtime ## Package runtime image for GH Actions testing + ./scripts/package-image-runtime + .PHONY: package-bundle package-bundle: build-binary ## Package the tarball bundle ./scripts/package-bundle diff --git a/pkg/rke2/rke2_linux_test.go b/pkg/rke2/rke2_linux_test.go index 555066e791..424e4fcb29 100644 --- a/pkg/rke2/rke2_linux_test.go +++ b/pkg/rke2/rke2_linux_test.go @@ -28,7 +28,6 @@ func Test_UnitInitExecutor(t *testing.T) { { name: "agent", args: args{ - clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil), cfg: Config{ ControlPlaneProbeConf: []string{"kube-proxy-startup-initial-delay-seconds=42"}, ControlPlaneResourceLimits: []string{"kube-proxy-cpu=123m"}, @@ -62,7 +61,6 @@ func Test_UnitInitExecutor(t *testing.T) { { name: "server", args: args{ - clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil), cfg: Config{ ControlPlaneProbeConf: []string{"kube-proxy-startup-initial-delay-seconds=123"}, ControlPlaneResourceLimits: []string{"kube-proxy-cpu=42m"}, @@ -96,7 +94,6 @@ func Test_UnitInitExecutor(t *testing.T) { { name: "bad probe conf", args: args{ - clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil), cfg: Config{ ControlPlaneProbeConf: []string{"kube-proxy-startup-initial-delay-seconds=-123"}, }, @@ -106,7 +103,6 @@ func Test_UnitInitExecutor(t *testing.T) { { name: "bad control plane limits", args: args{ - clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil), cfg: Config{ ControlPlaneResourceLimits: []string{"kube-proxy-cpu"}, }, @@ -116,7 +112,6 @@ func Test_UnitInitExecutor(t *testing.T) { { name: "bad control plane requests", args: args{ - clx: cli.NewContext(nil, flag.NewFlagSet("test", 0), nil), cfg: Config{ ControlPlaneResourceRequests: []string{"kube-proxy-memory"}, }, @@ -126,6 +121,10 @@ func Test_UnitInitExecutor(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // Override the pss location so we attempt to create a file that needs sudo, not what we are testing anyways + flagSet := flag.NewFlagSet("test", 0) + flagSet.String("pod-security-admission-config-file", "/tmp/pss.yaml", "") + tt.args.clx = cli.NewContext(nil, flagSet, nil) got, err := initExecutor(tt.args.clx, tt.args.cfg, tt.args.isServer) if (err != nil) != tt.wantErr { t.Errorf("initExecutor() error = %v, wantErr %v", err, tt.wantErr) diff --git a/scripts/build-binary b/scripts/build-binary index 406a47563e..dc5482f88a 100755 --- a/scripts/build-binary +++ b/scripts/build-binary @@ -13,6 +13,11 @@ else DEBUG_GO_GCFLAGS='-gcflags=all=-N -l' fi +if [ -n "${GOCOVER}" ]; then + GO_BUILDTAGS="${GO_BUILDTAGS} cover" + GO_BUILD_FLAGS="${GO_BUILD_FLAGS} -cover" +fi + REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .dirty; fi) RELEASE=${PROG}.${GOOS}-${GOARCH} diff --git a/scripts/package-image-runtime b/scripts/package-image-runtime new file mode 100755 index 0000000000..27ac09ddef --- /dev/null +++ b/scripts/package-image-runtime @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +cd $(dirname $0)/.. + +source ./scripts/version.sh + +if [ "${GOARCH}" == "s390x" ] || [ "${GOARCH}" == "arm64" ]; then + exit 0 +fi + +docker image save \ + ${REGISTRY}/${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION} | \ + zstd -T0 -16 -f --long=25 --no-progress - -o build/images/${PROG}-images.${PLATFORM}.tar.zst \ No newline at end of file diff --git a/scripts/publish-image-kubernetes b/scripts/publish-image-kubernetes deleted file mode 100755 index 278fdea1fd..0000000000 --- a/scripts/publish-image-kubernetes +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -ex - -cd $(dirname $0)/.. - -source ./scripts/version.sh - -docker image push ${REPO}/hardened-kubernetes:${DOCKERIZED_VERSION}-${GOARCH} diff --git a/tests/etcd_int_test.go b/tests/etcd_int_test.go deleted file mode 100644 index 22a83d2a9f..0000000000 --- a/tests/etcd_int_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package tests - -import ( - "regexp" - "strings" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/rancher/rke2/tests/util" -) - -var serverArgs = []string{"server"} - -var _ = Describe("etcd snapshots", func() { - BeforeEach(func() { - if !util.ServerArgsPresent(serverArgs) { - Skip("Test needs rke2 server with: " + strings.Join(serverArgs, " ")) - } - }) - When("a new etcd is created", func() { - It("starts up with no problems", func() { - Eventually(util.Rke2Ready(), "90s", "1s").Should(BeTrue()) - }) - It("saves an etcd snapshot", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "save")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - }) - It("list snapshots", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "ls")). - To(MatchRegexp(`:///var/lib/rancher/rke2/server/db/snapshots/on-demand`)) - }) - It("deletes a snapshot", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`on-demand[^\s]+`) - Expect(err).ToNot(HaveOccurred()) - snapshotName := reg.FindString(lsResult) - Expect(util.Rke2Cmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) - }) - }) - When("saving a custom name", func() { - It("saves an etcd snapshot with a custom name", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")). - To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/rke2/server/db/snapshots/ALIVEBEEF")) - }) - It("deletes that snapshot", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`ALIVEBEEF[^\s]+`) - Expect(err).ToNot(HaveOccurred()) - snapshotName := reg.FindString(lsResult) - Expect(util.Rke2Cmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) - }) - }) - When("using etcd snapshot prune", func() { - It("saves 3 different snapshots", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - time.Sleep(1 * time.Second) - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - time.Sleep(1 * time.Second) - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - time.Sleep(1 * time.Second) - }) - It("lists all 3 snapshots", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) - Expect(err).ToNot(HaveOccurred()) - sepLines := reg.FindAllString(lsResult, -1) - Expect(sepLines).To(HaveLen(3)) - }) - It("prunes snapshots down to 2", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "prune", "--snapshot-retention", "2", "--name", "PRUNE_TEST")). - To(BeEmpty()) - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) - Expect(err).ToNot(HaveOccurred()) - sepLines := reg.FindAllString(lsResult, -1) - Expect(sepLines).To(HaveLen(2)) - }) - It("cleans up remaining snapshots", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`PRUNE_TEST[^\s]+`) - Expect(err).ToNot(HaveOccurred()) - for _, snapshotName := range reg.FindAllString(lsResult, -1) { - Expect(util.Rke2Cmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) - } - }) - }) -}) - -func Test_IntegrationEtcd(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Etcd Suite") -} diff --git a/tests/install/centos-7/Vagrantfile b/tests/install/centos-7/Vagrantfile index eaa75a465f..a31b78f4cc 100644 --- a/tests/install/centos-7/Vagrantfile +++ b/tests/install/centos-7/Vagrantfile @@ -4,7 +4,7 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vm.box = "dweomer/centos-7.9-amd64" + config.vm.box = "generic/centos7" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false %w[libvirt virtualbox vmware_desktop].each do |p| @@ -14,6 +14,9 @@ Vagrant.configure("2") do |config| end end + # Load in helper functions + load "../install_util.rb" + external_env = "" ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} @@ -28,74 +31,15 @@ Vagrant.configure("2") do |config| YAML rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 500 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + checkRKE2Processes(test.vm) end config.vm.provision "install-packages", type: "shell", run: "once" do |sh| diff --git a/tests/install/install_util.rb b/tests/install/install_util.rb new file mode 100644 index 0000000000..114a71bd65 --- /dev/null +++ b/tests/install/install_util.rb @@ -0,0 +1,90 @@ +def waitForControlPlane(vm, box) + hostname = box.include?("opensuse") ? "$(hostnamectl --static)" : "$(hostname)" + vm.provision "rke2-wait-for-cp", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + echo 'Waiting for node (and static pods) to be ready ...' + time { + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-#{hostname} 2>/dev/null); do sleep 5; done' + } + kubectl get node,all -A -o wide + SHELL + end +end + + +def waitForCanal(vm) + vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def waitForCoreDNS(vm) + vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def waitForIngressNginx(vm) + vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def waitForMetricsServer(vm) + vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def checkRKE2Processes(vm) + vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep + SHELL + end +end + +def kubectlStatus(vm) + vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + kubectl get node,all -A -o wide + SHELL + end +end \ No newline at end of file diff --git a/tests/install/opensuse-leap/Vagrantfile b/tests/install/opensuse-leap/Vagrantfile index ed63d6074c..8b43db4638 100644 --- a/tests/install/opensuse-leap/Vagrantfile +++ b/tests/install/opensuse-leap/Vagrantfile @@ -4,7 +4,7 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vm.box = 'opensuse/Leap-15.4.x86_64' + config.vm.box = 'opensuse/Leap-15.5.x86_64' config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false %w[libvirt virtualbox vmware_desktop].each do |p| @@ -14,6 +14,9 @@ Vagrant.configure("2") do |config| end end + # Load in helper functions + load "../install_util.rb" + external_env = "" ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} @@ -28,74 +31,15 @@ Vagrant.configure("2") do |config| YAML rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + checkRKE2Processes(test.vm) end config.vm.provision "install-packages", type: "shell", run: "once" do |sh| diff --git a/tests/install/opensuse-microos/README.md b/tests/install/opensuse-microos/README.md deleted file mode 100644 index 500f0af4c3..0000000000 --- a/tests/install/opensuse-microos/README.md +++ /dev/null @@ -1,32 +0,0 @@ -RKE2 Install on MicroOS ---- - -Asserting correctness of the RKE2 installer script using [openSUSE MicroOS](https://microos.opensuse.org/) -as a stand-in for [SUSE Linux Enterprise Micro](https://www.suse.com/products/micro/). - -### Testing With Vagrant - -The [Vagrant box](https://app.vagrantup.com/dweomer/boxes/microos.amd64) used for this test supports these providers: -- `libvirt` -- `virtualbox` (the default for most installations, including `macos-12` github actions runners) -- `vmware_desktop` - -To spin up a VM to test a locally modified `install.sh`: -```shell -# make sure the vagrant-reload plugin is installed, one-time only -vagrant plugin install vagrant-reload -``` -```shell -vagrant up -``` - -See also: -- [developer-docs/testing.md](../../../developer-docs/testing.md#environment-variables) - -### Vagrant Reload Plugin - -The MicroOS guest leverages [transactional updates](https://documentation.suse.com/sles/15-SP1/html/SLES-all/cha-transactional-updates.html) -for most persistent mutations of the installation (typically involving the `/usr` partition) which requires a reboot to -take effect. The `vagrant-reload` provisioner plugin is used for this because the implementation of the [`reboot` option](https://www.vagrantup.com/docs/provisioning/shell#reboot) -of the built-in [Vagrant Shell Provisioner](https://www.vagrantup.com/docs/provisioning/shell) is unreliable -(especially if used more than once per provisioning run). diff --git a/tests/install/opensuse-microos/Vagrantfile b/tests/install/opensuse-microos/Vagrantfile deleted file mode 100644 index d02e504719..0000000000 --- a/tests/install/opensuse-microos/Vagrantfile +++ /dev/null @@ -1,140 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' - -Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-reload' => {}, - } - config.vm.box = "dweomer/microos.amd64" - config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds - config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false - %w[libvirt virtualbox vmware_desktop].each do |p| - config.vm.provider p do |v, o| - v.cpus = ENV['TEST_VM_CPUS'] || 2 - v.memory = ENV['TEST_VM_MEMORY'] || 3072 - end - end - - external_env = "" - ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} - - config.vm.define "install-microos", primary: true do |test| - test.vm.hostname = 'smoke' - test.vm.provision 'rke2-upload-installer', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' - test.vm.provision"rke2-install", type: 'rke2', run: "once" do |rke2| - rke2.installer_url = 'file:///home/vagrant/install.sh' - rke2.env = %W[ #{external_env} INSTALL_RKE2_TYPE=server] - rke2.config = <<~YAML - token: 'vagrant' - YAML - rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end - end - - config.vm.provision "install-packages", type: "shell", run: "once" do |sh| - sh.upload_path = "/tmp/vagrant-install-packages" - sh.env = { - 'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'], - } - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eux -o pipefail - transactional-update --no-selfupdate -d pkg install -y --allow-unsigned-rpm \ - curl \ - iptables \ - less \ - lsof \ - socat \ - ${INSTALL_PACKAGES} - SHELL - end - config.vm.provision "install-packages-reload", type: "reload", run: "once" - config.vm.provision "selinux-status", type: "shell", run: "once", inline: "sestatus -v" - config.vm.provision "rke2-profile-env", type: "shell", run: "once" do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - cat <<-EOF > /etc/profile.d/rke2.sh -export KUBECONFIG=/etc/rancher/rke2/rke2.yaml PATH=/usr/local/bin:$PATH:/var/lib/rancher/rke2/bin -EOF - SHELL - end - -end diff --git a/tests/install/rocky-8/Vagrantfile b/tests/install/rocky-8/Vagrantfile index d7938edfad..02ec340aa6 100644 --- a/tests/install/rocky-8/Vagrantfile +++ b/tests/install/rocky-8/Vagrantfile @@ -14,6 +14,9 @@ Vagrant.configure("2") do |config| end end + # Load in helper functions + load "../install_util.rb" + external_env = "" ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} @@ -29,74 +32,15 @@ Vagrant.configure("2") do |config| YAML rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + checkRKE2Processes(test.vm) end config.vm.provision "install-packages", type: "shell", run: "once" do |sh| diff --git a/tests/install/ubuntu-focal/README.md b/tests/install/ubuntu-2004/README.md similarity index 93% rename from tests/install/ubuntu-focal/README.md rename to tests/install/ubuntu-2004/README.md index 2b94b7f9d7..9b6719ad65 100644 --- a/tests/install/ubuntu-focal/README.md +++ b/tests/install/ubuntu-2004/README.md @@ -1,4 +1,4 @@ -RKE2 Install on Ubuntu Focal Fossa +RKE2 Install on Ubuntu 20.04 Focal Fossa --- Asserting correctness of the RKE2 installer script on [Ubuntu 20.04](https://releases.ubuntu.com/20.04/). diff --git a/tests/install/ubuntu-2004/Vagrantfile b/tests/install/ubuntu-2004/Vagrantfile new file mode 100644 index 0000000000..92ea543007 --- /dev/null +++ b/tests/install/ubuntu-2004/Vagrantfile @@ -0,0 +1,80 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' + +Vagrant.configure("2") do |config| + config.vm.box = "generic/ubuntu2004" + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false + %w[libvirt virtualbox vmware_desktop].each do |p| + config.vm.provider p do |v, o| + v.cpus = ENV['TEST_VM_CPUS'] || 2 + v.memory = ENV['TEST_VM_MEMORY'] || 3072 + end + end + + # Load in helper functions + load "../install_util.rb" + + external_env = "" + ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} + + config.vm.define "install-ubuntu-2004", primary: true do |test| + test.vm.hostname = 'smoke' + test.vm.provision 'rke2-upload-installer', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' + test.vm.provision"rke2-install", type: 'rke2', run: "once" do |rke2| + rke2.installer_url = 'file:///home/vagrant/install.sh' + rke2.env = %W[ #{external_env} INSTALL_RKE2_TYPE=server] + rke2.config = <<~YAML + token: 'vagrant' + YAML + rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'etcd|kube|rke2|container|confined' | grep -v grep + SHELL + end + end + + config.vm.provision "install-packages", type: "shell", run: "once" do |sh| + sh.upload_path = "/tmp/vagrant-install-packages" + sh.env = { + 'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'], + } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + apt-get -y update + apt-get -y install \ + curl \ + iptables \ + less \ + lsof \ + netcat \ + socat \ + ${INSTALL_PACKAGES} + SHELL + end + + config.vm.provision "rke2-profile-env", type: "shell", run: "once" do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + cat <<-EOF > /etc/profile.d/rke2.sh +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml PATH=/usr/local/bin:$PATH:/var/lib/rancher/rke2/bin +EOF + SHELL + end + +end diff --git a/tests/install/ubuntu-focal/Vagrantfile b/tests/install/ubuntu-focal/Vagrantfile deleted file mode 100644 index 63ad652759..0000000000 --- a/tests/install/ubuntu-focal/Vagrantfile +++ /dev/null @@ -1,130 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' - -Vagrant.configure("2") do |config| - config.vm.box = "generic/ubuntu2004" - config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds - config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false - %w[libvirt virtualbox vmware_desktop].each do |p| - config.vm.provider p do |v, o| - v.cpus = ENV['TEST_VM_CPUS'] || 2 - v.memory = ENV['TEST_VM_MEMORY'] || 3072 - end - end - - external_env = "" - ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} - - config.vm.define "install-ubuntu-focal", primary: true do |test| - test.vm.hostname = 'smoke' - test.vm.provision 'rke2-upload-installer', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' - test.vm.provision"rke2-install", type: 'rke2', run: "once" do |rke2| - rke2.installer_url = 'file:///home/vagrant/install.sh' - rke2.env = %W[ #{external_env} INSTALL_RKE2_TYPE=server] - rke2.config = <<~YAML - token: 'vagrant' - YAML - rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|confined' | grep -v grep - SHELL - end - end - - config.vm.provision "install-packages", type: "shell", run: "once" do |sh| - sh.upload_path = "/tmp/vagrant-install-packages" - sh.env = { - 'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'], - } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - apt-get -y update - apt-get -y install \ - curl \ - iptables \ - less \ - lsof \ - netcat \ - socat \ - ${INSTALL_PACKAGES} - SHELL - end - - config.vm.provision "rke2-profile-env", type: "shell", run: "once" do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - cat <<-EOF > /etc/profile.d/rke2.sh -export KUBECONFIG=/etc/rancher/rke2/rke2.yaml PATH=/usr/local/bin:$PATH:/var/lib/rancher/rke2/bin -EOF - SHELL - end - -end diff --git a/tests/install/windows-2019/Vagrantfile b/tests/install/windows-2019/Vagrantfile index 8991aa0890..68a7d7e04e 100644 --- a/tests/install/windows-2019/Vagrantfile +++ b/tests/install/windows-2019/Vagrantfile @@ -5,7 +5,7 @@ ENV['TEST_INSTALL_PS1'] ||= '../../../install.ps1' Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-reload"] - config.vm.box = "gusztavvargadr/windows-server-2019-standard" + config.vm.box = "jborean93/WindowsServer2019" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true %w[libvirt virtualbox hyperv].each do |p| diff --git a/tests/integration/etcdsnapshot/etcd_int_test.go b/tests/integration/etcdsnapshot/etcd_int_test.go new file mode 100644 index 0000000000..5c359f6556 --- /dev/null +++ b/tests/integration/etcdsnapshot/etcd_int_test.go @@ -0,0 +1,134 @@ +package tests + +import ( + "fmt" + "os" + "regexp" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + testutil "github.com/rancher/rke2/tests/integration" + "github.com/sirupsen/logrus" +) + +var serverArgs = []string{""} +var serverLog *os.File +var testLock int + +var _ = BeforeSuite(func() { + var err error + testLock, err = testutil.AcquireTestLock() + Expect(err).ToNot(HaveOccurred()) + serverLog, err = testutil.StartServer(serverArgs...) + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = Describe("etcd snapshots", Ordered, func() { + When("a new etcd snapshot is created", func() { + It("starts up with no problems", func() { + Eventually(func() error { + err := testutil.ServerReady() + if err != nil { + logrus.Info(err) + } + return err + }, "240s", "15s").Should(Succeed()) + }) + It("saves an etcd snapshot", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "save")). + To(And(ContainSubstring("Snapshot on-demand-"), ContainSubstring(" saved."))) + }) + It("list snapshots", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "ls")). + To(MatchRegexp(`:///var/lib/rancher/rke2/server/db/snapshots/on-demand`)) + }) + It("deletes a snapshot", func() { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`on-demand[^\s]+`) + Expect(err).ToNot(HaveOccurred()) + snapshotName := reg.FindString(lsResult) + Expect(testutil.RKE2Cmd("etcd-snapshot", "delete", snapshotName)). + To(ContainSubstring(fmt.Sprintf("Snapshot %s deleted", snapshotName))) + }) + }) + When("saving a custom name", func() { + It("saves an etcd snapshot with a custom name", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")). + To(And(ContainSubstring("Snapshot ALIVEBEEF-"), ContainSubstring(" saved."))) + }) + It("deletes that snapshot", func() { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`ALIVEBEEF[^\s]+`) + Expect(err).ToNot(HaveOccurred()) + snapshotName := reg.FindString(lsResult) + Expect(testutil.RKE2Cmd("etcd-snapshot", "delete", snapshotName)). + To(ContainSubstring("Snapshot %s deleted.", snapshotName)) + }) + }) + When("using etcd snapshot prune", func() { + It("saves 3 different snapshots", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). + To(And(ContainSubstring("Snapshot PRUNE_TEST-"), ContainSubstring(" saved."))) + time.Sleep(1 * time.Second) + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). + To(And(ContainSubstring("Snapshot PRUNE_TEST-"), ContainSubstring(" saved."))) + time.Sleep(1 * time.Second) + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). + To(And(ContainSubstring("Snapshot PRUNE_TEST-"), ContainSubstring(" saved."))) + time.Sleep(1 * time.Second) + }) + It("lists all 3 snapshots", func() { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) + Expect(err).ToNot(HaveOccurred()) + sepLines := reg.FindAllString(lsResult, -1) + Expect(sepLines).To(HaveLen(3)) + }) + It("prunes snapshots down to 2", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "prune", "--snapshot-retention", "2", "--name", "PRUNE_TEST")). + To(ContainSubstring(" deleted.")) + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) + Expect(err).ToNot(HaveOccurred()) + sepLines := reg.FindAllString(lsResult, -1) + Expect(sepLines).To(HaveLen(2)) + }) + It("cleans up remaining snapshots", func() { + Eventually(func(g Gomega) { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + g.Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`PRUNE_TEST[^\s]+`) + g.Expect(err).ToNot(HaveOccurred()) + for _, snapshotName := range reg.FindAllString(lsResult, -1) { + g.Expect(testutil.RKE2Cmd("etcd-snapshot", "delete", snapshotName)). + To(ContainSubstring("Snapshot %s deleted.", snapshotName)) + } + }, "20s", "5s").Should(Succeed()) + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if failed { + testutil.SaveLog(serverLog, false) + serverLog = nil + } + Expect(testutil.KillServer(serverLog)).To(Succeed()) + Expect(testutil.Cleanup(testLock)).To(Succeed()) +}) + +func Test_IntegrationEtcd(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Etcd Suite") +} diff --git a/tests/integration/integration.go b/tests/integration/integration.go new file mode 100644 index 0000000000..3aa115148b --- /dev/null +++ b/tests/integration/integration.go @@ -0,0 +1,322 @@ +package util + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/user" + "strings" + "time" + + "github.com/k3s-io/k3s/pkg/flock" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const lockFile = "/tmp/rke2-test.lock" + +func findFile(file string) (string, error) { + i := 0 + for { + if i > 3 { + return "", fmt.Errorf("could not find %s", file) + } + if _, err := os.Stat(file); err != nil { + file = "../" + file + continue + } + i++ + break + } + + return file, nil +} + +func findRKE2Executable() (string, error) { + return findFile("bin/rke2") +} + +func findBundleExecutable(binary string) (string, error) { + return findFile("bundle/bin/" + binary) +} + +// RKE2Cmd launches the provided RKE2 command via exec. Command blocks until finished. +// Kubectl commands can also be passed. +// Command output from both Stderr and Stdout is provided via string. +// cmdEx1, err := RKE2Cmd("etcd-snapshot", "ls") +// cmdEx2, err := RKE2Cmd("kubectl", "get", "pods", "-A") +func RKE2Cmd(cmdName string, cmdArgs ...string) (string, error) { + if cmdName == "kubectl" { + byteOut, err := exec.Command(cmdName, cmdArgs...).CombinedOutput() + return string(byteOut), err + } + rke2Bin, err := findRKE2Executable() + if err != nil { + return "", err + } + rke2Cmd := append([]string{cmdName}, cmdArgs...) + byteOut, err := exec.Command(rke2Bin, rke2Cmd...).CombinedOutput() + return string(byteOut), err +} + +// isRoot return true if the user is root (UID 0) +func isRoot() bool { + currentUser, err := user.Current() + if err != nil { + return false + } + return currentUser.Uid == "0" +} + +func AcquireTestLock() (int, error) { + logrus.Info("waiting to get test lock") + return flock.Acquire(lockFile) +} + +// StartServer acquires an exclusive lock on a temporary file, then launches a RKE2 cluster +// with the provided arguments. Subsequent/parallel calls to this function will block until +// the original lock is cleared using RKE2KillServer +// A file is returned to capture the output of the RKE2 server for debugging +func StartServer(inputArgs ...string) (*os.File, error) { + if !isRoot() { + return nil, errors.New("integration tests must be run as sudo/root") + } + + // Prepary RKE2 images if they are not present + _, err := os.Stat("/var/lib/rancher/rke2/agent/images") + if err != nil { + os.MkdirAll("/var/lib/rancher/rke2/agent", 0755) + imageDir, err := findFile("build/images") + if err != nil { + return nil, err + } + cmd := exec.Command("cp", "-r", imageDir, "/var/lib/rancher/rke2/agent/images") + if res, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("error copying images: %s: %w", res, err) + } + } + rke2Bin, err := findRKE2Executable() + if err != nil { + return nil, err + } + rke2Cmd := append([]string{"server"}, inputArgs...) + cmd := exec.Command(rke2Bin, rke2Cmd...) + // Pipe output to a file for debugging later + f, err := os.Create("./r2log.txt") + if err != nil { + return nil, err + } + cmd.Stderr = f + err = cmd.Start() + return f, err +} + +// KillServer terminates the running RKE2 server and its children using rke2-killall.sh +func KillServer(log *os.File) error { + + killall, err := findBundleExecutable("rke2-killall.sh") + if err != nil { + return err + } + cmd := exec.Command(killall) + if res, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error killing rke2 server: %s: %w", res, err) + } + + if log != nil { + log.Close() + os.Remove(log.Name()) + } + time.Sleep(2 * time.Second) + return nil +} + +// SaveLog closes the server log file and optionally dumps the contents to stdout +func SaveLog(log *os.File, dump bool) error { + log.Close() + if !dump { + return nil + } + log, err := os.Open(log.Name()) + if err != nil { + return err + } + defer log.Close() + b, err := io.ReadAll(log) + if err != nil { + return err + } + fmt.Printf("Server Log Dump:\n\n%s\n\n", b) + return nil +} + +// Cleanup unlocks the test-lock and run the rke2-uninstall.sh script +// with one exception: we save and then restore the agent/images directory, +// for use with the next test. +func Cleanup(rke2TestLock int) error { + // Save the agent/images directory + cmd := exec.Command("cp", "-r", "/var/lib/rancher/rke2/agent/images", "/tmp/images-backup") + if res, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error backing up images directory: %s: %w", res, err) + } + + uninstall, err := findBundleExecutable("rke2-uninstall.sh") + if err != nil { + return err + } + // We don't care about the return value of the uninstall script, + // as it will always return an error because no rk2e service is running + exec.Command(uninstall).Run() + + // Restore the agent/images directory + if err := os.MkdirAll("/var/lib/rancher/rke2/agent", 0755); err != nil { + return fmt.Errorf("failed to make agent directory: %w", err) + } + cmd = exec.Command("mv", "/tmp/images-backup", "/var/lib/rancher/rke2/agent/images") + if res, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error restoring images directory: %s: %w", res, err) + } + if rke2TestLock != -1 { + return flock.Release(rke2TestLock) + } + return nil +} + +// ServerReady checks if the server is ready by checking the status of the pods +// and deployments that are required for the server to be operational +// On success, returns nil +func ServerReady() error { + hn, err := os.Hostname() + if err != nil { + return err + } + podsToCheck := []string{ + "etcd-" + hn, + "kube-apiserver-" + hn, + "kube-proxy-" + hn, + "kube-scheduler-" + hn, + } + deploymentsToCheck := []string{ + "rke2-coredns-rke2-coredns", + "rke2-metrics-server", + "rke2-snapshot-controller", + } + + pods, err := ParsePods("kube-system", metav1.ListOptions{}) + if err != nil { + return err + } + for _, pod := range podsToCheck { + ready := false + for _, p := range pods { + if p.Name == pod && p.Status.Phase == "Running" { + ready = true + } + } + if !ready { + return fmt.Errorf("pod %s is not ready", pod) + } + } + + return CheckDeployments(deploymentsToCheck) +} + +func ParsePods(namespace string, opts metav1.ListOptions) ([]corev1.Pod, error) { + clientSet, err := k8sClient() + if err != nil { + return nil, err + } + pods, err := clientSet.CoreV1().Pods(namespace).List(context.Background(), opts) + if err != nil { + return nil, err + } + + return pods.Items, nil +} + +// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error +func CheckDeployments(deployments []string) error { + + deploymentSet := make(map[string]bool) + for _, d := range deployments { + deploymentSet[d] = false + } + + client, err := k8sClient() + if err != nil { + return err + } + deploymentList, err := client.AppsV1().Deployments("kube-system").List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas { + deploymentSet[deployment.Name] = true + } + } + for d, found := range deploymentSet { + if !found { + return fmt.Errorf("deployment %s is not ready", d) + } + } + + return nil +} + +func contains(source []string, target string) bool { + for _, s := range source { + if s == target { + return true + } + } + return false +} + +// ServerArgsPresent checks if the given arguments are found in the running RKE2 server +func ServerArgsPresent(neededArgs []string) bool { + currentArgs, err := ServerArgs() + if err != nil { + logrus.Error(err) + return false + } + for _, arg := range neededArgs { + if !contains(currentArgs, arg) { + return false + } + } + return true +} + +// ServerArgs returns the list of arguments that the RKE2 server launched with +func ServerArgs() ([]string, error) { + results, err := RKE2Cmd("kubectl", "get", "nodes", "-o", `jsonpath='{.items[0].metadata.annotations.rke2\.io/node-args}'`) + if err != nil { + return nil, err + } + res := strings.ReplaceAll(results, "'", "") + var args []string + if err := json.Unmarshal([]byte(res), &args); err != nil { + return nil, err + } + return args, nil +} + +func k8sClient() (*kubernetes.Clientset, error) { + config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/rke2/rke2.yaml") + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return clientSet, nil +} diff --git a/tests/util/cmd.go b/tests/util/cmd.go deleted file mode 100644 index dd0fa64322..0000000000 --- a/tests/util/cmd.go +++ /dev/null @@ -1,101 +0,0 @@ -package util - -import ( - "encoding/json" - "os" - "os/exec" - "regexp" - "strings" - - "github.com/sirupsen/logrus" -) - -func findRke2Executable() string { - rke2Bin := "bin/rke2" - for { - _, err := os.Stat(rke2Bin) - if err != nil { - rke2Bin = "../" + rke2Bin - continue - } - break - } - return rke2Bin -} - -// Rke2Cmd launches the provided Rke2 command via exec. Command blocks until finished. -// Kubectl commands can also be passed. -// Command output from both Stderr and Stdout is provided via string. -// cmdEx1, err := Rke2Cmd("etcd-snapshot", "ls") -// cmdEx2, err := Rke2Cmd("kubectl", "get", "pods", "-A") -func Rke2Cmd(cmdName string, cmdArgs ...string) (string, error) { - if cmdName == "kubectl" { - byteOut, err := exec.Command(cmdName, cmdArgs...).CombinedOutput() - return string(byteOut), err - } - rke2Bin := findRke2Executable() - rke2Cmd := append([]string{cmdName}, cmdArgs...) - byteOut, err := exec.Command(rke2Bin, rke2Cmd...).CombinedOutput() - return string(byteOut), err -} - -func Rke2Ready() bool { - podsToCheck := []string{ - "etcd-rke2-server", - "kube-apiserver-rke2-server", - "kube-proxy-rke2-server", - "kube-scheduler-rke2-server", - "rke2-coredns-rke2-coredns", - } - pods, err := Rke2Cmd("kubectl", "get", "pods", "-A") - if err != nil { - return false - } - for _, pod := range podsToCheck { - reg := pod + ".+Running" - match, err := regexp.MatchString(reg, pods) - if !match || err != nil { - logrus.Error(err) - return false - } - } - return true -} - -func contains(source []string, target string) bool { - for _, s := range source { - if s == target { - return true - } - } - return false -} - -// ServerArgsPresent checks if the given arguments are found in the running k3s server -func ServerArgsPresent(neededArgs []string) bool { - currentArgs, err := Rke2ServerArgs() - if err != nil { - logrus.Error(err) - return false - } - for _, arg := range neededArgs { - if !contains(currentArgs, arg) { - return false - } - } - return true -} - -// Rke2ServerArgs returns the list of arguments that the rke2 server launched with -func Rke2ServerArgs() ([]string, error) { - results, err := Rke2Cmd("kubectl", "get", "nodes", "-o", `jsonpath='{.items[0].metadata.annotations.rke2\.io/node-args}'`) - if err != nil { - return nil, err - } - res := strings.ReplaceAll(results, "'", "") - var args []string - if err := json.Unmarshal([]byte(res), &args); err != nil { - return nil, err - } - return args, nil -}