diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml new file mode 100644 index 0000000000..fddde276d5 --- /dev/null +++ b/.github/workflows/integration.yaml @@ -0,0 +1,114 @@ +name: Integration Test Coverage +on: + push: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/integration**" + - ".github/**" + - "!.github/workflows/integration.yaml" + pull_request: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/integration**" + - "!tests/e2e**" + - ".github/**" + - "!.github/workflows/integration.yaml" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + build: + name: Build RKE2 Images and Binary + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache-dependency-path: | + **/go.sum + **/go.mod + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install OS Packages + run: sudo apt-get install -y libarchive-tools + - name: Build RKE2 Binary and Runtime Image + run: | + GOCOVER=true make build-binary + make package-image-runtime + cp ./bin/rke2 ./build/images/rke2-binary + # Can only upload from a single path, so we need to copy the binary to the image directory + - name: Upload RKE2 Binary and Runtime Image + uses: actions/upload-artifact@v4 + with: + name: rke2-test-artifacts + path: ./build/images/* + test: + needs: build + name: Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + itest: [etcdsnapshot] + max-parallel: 3 + env: + GOCOVERDIR: /tmp/rke2cov + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache-dependency-path: | + **/go.sum + **/go.mod + - name: Setup Build Directories + run: | + mkdir -p ./bin ./build/images + - name: Download RKE2 Binary and Runtime Image + uses: actions/download-artifact@v4 + with: + name: rke2-test-artifacts + path: ./build/images + - name: Setup Binary + run: | + mv ./build/images/rke2-binary ./bin/rke2 + chmod +x ./bin/rke2 + - name: Run Integration Tests + run: | + mkdir -p $GOCOVERDIR + sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration + - name: Generate coverage report + run: go tool covdata textfmt -i $GOCOVERDIR -o ${{ matrix.itest }}.out + - name: Upload Results To Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./${{ matrix.itest }}.out + flags: inttests # optional + verbose: true # optional (default = false) + - name: On Failure, Dump Server Logs + if: ${{ failure() }} + run: cat ./tests/integration/${{ matrix.itest }}/r2log.txt + - name: On Failure, Launch Debug Session + uses: dereknola/action-upterm@main + if: ${{ failure() }} + with: + ## If no one connects after 5 minutes, shut down server. + wait-timeout-minutes: 5 + limit-access-to-actor: true diff --git a/Dockerfile b/Dockerfile index 91082f76d6..070a4c73e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN zypper install -y systemd-rpm-macros # Dapper/Drone/CI environment FROM build AS dapper -ENV DAPPER_ENV GODEBUG REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY +ENV DAPPER_ENV GODEBUG GOCOVER REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY ARG DAPPER_HOST_ARCH ENV ARCH $DAPPER_HOST_ARCH ENV DAPPER_OUTPUT ./dist ./bin ./build diff --git a/Makefile b/Makefile index 73444032fa..24dc8396f6 100644 --- a/Makefile +++ b/Makefile @@ -47,18 +47,10 @@ build-images: ## Build all images and image tarballs build-windows-images: ## Build only the Windows images and tarballs (including airgap) ./scripts/build-windows-images -.PHONY: build-image-kubernetes -build-image-kubernetes: ## Build the kubernetes image - ./scripts/build-image-kubernetes - .PHONY: build-image-runtime build-image-runtime: ## Build the runtime image ./scripts/build-image-runtime -.PHONY: publish-image-kubernetes -publish-image-kubernetes: build-image-kubernetes - ./scripts/publish-image-kubernetes - .PHONY: publish-image-runtime publish-image-runtime: build-image-runtime ./scripts/publish-image-runtime @@ -136,6 +128,10 @@ package-images: build-images ## Package docker images for airgap environment package-windows-images: build-windows-images ## Package Windows crane images for airgap environment ./scripts/package-windows-images +.PHONY: package-image-runtime +package-image-runtime: build-image-runtime ## Package runtime image for GH Actions testing + ./scripts/package-image-runtime + .PHONY: package-bundle package-bundle: build-binary ## Package the tarball bundle ./scripts/package-bundle diff --git a/scripts/build-binary b/scripts/build-binary index 406a47563e..dc5482f88a 100755 --- a/scripts/build-binary +++ b/scripts/build-binary @@ -13,6 +13,11 @@ else DEBUG_GO_GCFLAGS='-gcflags=all=-N -l' fi +if [ -n "${GOCOVER}" ]; then + GO_BUILDTAGS="${GO_BUILDTAGS} cover" + GO_BUILD_FLAGS="${GO_BUILD_FLAGS} -cover" +fi + REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .dirty; fi) RELEASE=${PROG}.${GOOS}-${GOARCH} diff --git a/scripts/package-image-runtime b/scripts/package-image-runtime new file mode 100755 index 0000000000..27ac09ddef --- /dev/null +++ b/scripts/package-image-runtime @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +cd $(dirname $0)/.. + +source ./scripts/version.sh + +if [ "${GOARCH}" == "s390x" ] || [ "${GOARCH}" == "arm64" ]; then + exit 0 +fi + +docker image save \ + ${REGISTRY}/${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION} | \ + zstd -T0 -16 -f --long=25 --no-progress - -o build/images/${PROG}-images.${PLATFORM}.tar.zst \ No newline at end of file diff --git a/scripts/publish-image-kubernetes b/scripts/publish-image-kubernetes deleted file mode 100755 index 278fdea1fd..0000000000 --- a/scripts/publish-image-kubernetes +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -ex - -cd $(dirname $0)/.. - -source ./scripts/version.sh - -docker image push ${REPO}/hardened-kubernetes:${DOCKERIZED_VERSION}-${GOARCH} diff --git a/tests/etcd_int_test.go b/tests/etcd_int_test.go deleted file mode 100644 index 22a83d2a9f..0000000000 --- a/tests/etcd_int_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package tests - -import ( - "regexp" - "strings" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/rancher/rke2/tests/util" -) - -var serverArgs = []string{"server"} - -var _ = Describe("etcd snapshots", func() { - BeforeEach(func() { - if !util.ServerArgsPresent(serverArgs) { - Skip("Test needs rke2 server with: " + strings.Join(serverArgs, " ")) - } - }) - When("a new etcd is created", func() { - It("starts up with no problems", func() { - Eventually(util.Rke2Ready(), "90s", "1s").Should(BeTrue()) - }) - It("saves an etcd snapshot", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "save")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - }) - It("list snapshots", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "ls")). - To(MatchRegexp(`:///var/lib/rancher/rke2/server/db/snapshots/on-demand`)) - }) - It("deletes a snapshot", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`on-demand[^\s]+`) - Expect(err).ToNot(HaveOccurred()) - snapshotName := reg.FindString(lsResult) - Expect(util.Rke2Cmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) - }) - }) - When("saving a custom name", func() { - It("saves an etcd snapshot with a custom name", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")). - To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/rke2/server/db/snapshots/ALIVEBEEF")) - }) - It("deletes that snapshot", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`ALIVEBEEF[^\s]+`) - Expect(err).ToNot(HaveOccurred()) - snapshotName := reg.FindString(lsResult) - Expect(util.Rke2Cmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) - }) - }) - When("using etcd snapshot prune", func() { - It("saves 3 different snapshots", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - time.Sleep(1 * time.Second) - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - time.Sleep(1 * time.Second) - Expect(util.Rke2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). - To(ContainSubstring("Saving current etcd snapshot set to rke2-etcd-snapshots")) - time.Sleep(1 * time.Second) - }) - It("lists all 3 snapshots", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) - Expect(err).ToNot(HaveOccurred()) - sepLines := reg.FindAllString(lsResult, -1) - Expect(sepLines).To(HaveLen(3)) - }) - It("prunes snapshots down to 2", func() { - Expect(util.Rke2Cmd("etcd-snapshot", "prune", "--snapshot-retention", "2", "--name", "PRUNE_TEST")). - To(BeEmpty()) - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) - Expect(err).ToNot(HaveOccurred()) - sepLines := reg.FindAllString(lsResult, -1) - Expect(sepLines).To(HaveLen(2)) - }) - It("cleans up remaining snapshots", func() { - lsResult, err := util.Rke2Cmd("etcd-snapshot", "ls") - Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`PRUNE_TEST[^\s]+`) - Expect(err).ToNot(HaveOccurred()) - for _, snapshotName := range reg.FindAllString(lsResult, -1) { - Expect(util.Rke2Cmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) - } - }) - }) -}) - -func Test_IntegrationEtcd(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Etcd Suite") -} diff --git a/tests/integration/etcdsnapshot/etcd_int_test.go b/tests/integration/etcdsnapshot/etcd_int_test.go new file mode 100644 index 0000000000..79160a287b --- /dev/null +++ b/tests/integration/etcdsnapshot/etcd_int_test.go @@ -0,0 +1,134 @@ +package tests + +import ( + "fmt" + "os" + "regexp" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + testutil "github.com/rancher/rke2/tests/integration" + "github.com/sirupsen/logrus" +) + +var serverArgs = []string{""} +var serverLog *os.File +var testLock int + +var _ = BeforeSuite(func() { + var err error + testLock, err = testutil.AcquireTestLock() + Expect(err).ToNot(HaveOccurred()) + serverLog, err = testutil.StartServer(serverArgs...) + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = Describe("etcd snapshots", Ordered, func() { + When("a new etcd snapshot is created", func() { + It("starts up with no problems", func() { + Eventually(func() error { + err := testutil.ServerReady() + if err != nil { + logrus.Info(err) + } + return err + }, "240s", "15s").Should(Succeed()) + }) + It("saves an etcd snapshot", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "save")). + To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/rke2/server/db/snapshots/on-demand")) + }) + It("list snapshots", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "ls")). + To(MatchRegexp(`:///var/lib/rancher/rke2/server/db/snapshots/on-demand`)) + }) + It("deletes a snapshot", func() { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`on-demand[^\s]+`) + Expect(err).ToNot(HaveOccurred()) + snapshotName := reg.FindString(lsResult) + Expect(testutil.RKE2Cmd("etcd-snapshot", "delete", snapshotName)). + To(ContainSubstring(fmt.Sprintf("Snapshot %s deleted locally", snapshotName))) + }) + }) + When("saving a custom name", func() { + It("saves an etcd snapshot with a custom name", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")). + To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/rke2/server/db/snapshots/ALIVEBEEF")) + }) + It("deletes that snapshot", func() { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`ALIVEBEEF[^\s]+`) + Expect(err).ToNot(HaveOccurred()) + snapshotName := reg.FindString(lsResult) + Expect(testutil.RKE2Cmd("etcd-snapshot", "delete", snapshotName)). + To(ContainSubstring(fmt.Sprintf("Snapshot %s deleted locally", snapshotName))) + }) + }) + When("using etcd snapshot prune", func() { + It("saves 3 different snapshots", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). + To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST")) + time.Sleep(1 * time.Second) + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). + To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST")) + time.Sleep(1 * time.Second) + Expect(testutil.RKE2Cmd("etcd-snapshot", "save", "--name", "PRUNE_TEST")). + To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST")) + time.Sleep(1 * time.Second) + }) + It("lists all 3 snapshots", func() { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) + Expect(err).ToNot(HaveOccurred()) + sepLines := reg.FindAllString(lsResult, -1) + Expect(sepLines).To(HaveLen(3)) + }) + It("prunes snapshots down to 2", func() { + Expect(testutil.RKE2Cmd("etcd-snapshot", "prune", "--snapshot-retention", "2", "--name", "PRUNE_TEST")). + To(ContainSubstring("Applying snapshot retention=2")) + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`:///var/lib/rancher/rke2/server/db/snapshots/PRUNE_TEST`) + Expect(err).ToNot(HaveOccurred()) + sepLines := reg.FindAllString(lsResult, -1) + Expect(sepLines).To(HaveLen(2)) + }) + It("cleans up remaining snapshots", func() { + Eventually(func(g Gomega) { + lsResult, err := testutil.RKE2Cmd("etcd-snapshot", "ls") + g.Expect(err).ToNot(HaveOccurred()) + reg, err := regexp.Compile(`PRUNE_TEST[^\s]+`) + g.Expect(err).ToNot(HaveOccurred()) + for _, snapshotName := range reg.FindAllString(lsResult, -1) { + g.Expect(testutil.RKE2Cmd("etcd-snapshot", "delete", snapshotName)). + To(ContainSubstring(fmt.Sprintf("Snapshot %s deleted locally", snapshotName))) + } + }, "20s", "5s").Should(Succeed()) + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if failed { + testutil.SaveLog(serverLog, false) + serverLog = nil + } + Expect(testutil.KillServer(serverLog)).To(Succeed()) + Expect(testutil.Cleanup(testLock)).To(Succeed()) +}) + +func Test_IntegrationEtcd(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Etcd Suite") +} diff --git a/tests/integration/integration.go b/tests/integration/integration.go new file mode 100644 index 0000000000..3aa115148b --- /dev/null +++ b/tests/integration/integration.go @@ -0,0 +1,322 @@ +package util + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/user" + "strings" + "time" + + "github.com/k3s-io/k3s/pkg/flock" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const lockFile = "/tmp/rke2-test.lock" + +func findFile(file string) (string, error) { + i := 0 + for { + if i > 3 { + return "", fmt.Errorf("could not find %s", file) + } + if _, err := os.Stat(file); err != nil { + file = "../" + file + continue + } + i++ + break + } + + return file, nil +} + +func findRKE2Executable() (string, error) { + return findFile("bin/rke2") +} + +func findBundleExecutable(binary string) (string, error) { + return findFile("bundle/bin/" + binary) +} + +// RKE2Cmd launches the provided RKE2 command via exec. Command blocks until finished. +// Kubectl commands can also be passed. +// Command output from both Stderr and Stdout is provided via string. +// cmdEx1, err := RKE2Cmd("etcd-snapshot", "ls") +// cmdEx2, err := RKE2Cmd("kubectl", "get", "pods", "-A") +func RKE2Cmd(cmdName string, cmdArgs ...string) (string, error) { + if cmdName == "kubectl" { + byteOut, err := exec.Command(cmdName, cmdArgs...).CombinedOutput() + return string(byteOut), err + } + rke2Bin, err := findRKE2Executable() + if err != nil { + return "", err + } + rke2Cmd := append([]string{cmdName}, cmdArgs...) + byteOut, err := exec.Command(rke2Bin, rke2Cmd...).CombinedOutput() + return string(byteOut), err +} + +// isRoot return true if the user is root (UID 0) +func isRoot() bool { + currentUser, err := user.Current() + if err != nil { + return false + } + return currentUser.Uid == "0" +} + +func AcquireTestLock() (int, error) { + logrus.Info("waiting to get test lock") + return flock.Acquire(lockFile) +} + +// StartServer acquires an exclusive lock on a temporary file, then launches a RKE2 cluster +// with the provided arguments. Subsequent/parallel calls to this function will block until +// the original lock is cleared using RKE2KillServer +// A file is returned to capture the output of the RKE2 server for debugging +func StartServer(inputArgs ...string) (*os.File, error) { + if !isRoot() { + return nil, errors.New("integration tests must be run as sudo/root") + } + + // Prepary RKE2 images if they are not present + _, err := os.Stat("/var/lib/rancher/rke2/agent/images") + if err != nil { + os.MkdirAll("/var/lib/rancher/rke2/agent", 0755) + imageDir, err := findFile("build/images") + if err != nil { + return nil, err + } + cmd := exec.Command("cp", "-r", imageDir, "/var/lib/rancher/rke2/agent/images") + if res, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("error copying images: %s: %w", res, err) + } + } + rke2Bin, err := findRKE2Executable() + if err != nil { + return nil, err + } + rke2Cmd := append([]string{"server"}, inputArgs...) + cmd := exec.Command(rke2Bin, rke2Cmd...) + // Pipe output to a file for debugging later + f, err := os.Create("./r2log.txt") + if err != nil { + return nil, err + } + cmd.Stderr = f + err = cmd.Start() + return f, err +} + +// KillServer terminates the running RKE2 server and its children using rke2-killall.sh +func KillServer(log *os.File) error { + + killall, err := findBundleExecutable("rke2-killall.sh") + if err != nil { + return err + } + cmd := exec.Command(killall) + if res, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error killing rke2 server: %s: %w", res, err) + } + + if log != nil { + log.Close() + os.Remove(log.Name()) + } + time.Sleep(2 * time.Second) + return nil +} + +// SaveLog closes the server log file and optionally dumps the contents to stdout +func SaveLog(log *os.File, dump bool) error { + log.Close() + if !dump { + return nil + } + log, err := os.Open(log.Name()) + if err != nil { + return err + } + defer log.Close() + b, err := io.ReadAll(log) + if err != nil { + return err + } + fmt.Printf("Server Log Dump:\n\n%s\n\n", b) + return nil +} + +// Cleanup unlocks the test-lock and run the rke2-uninstall.sh script +// with one exception: we save and then restore the agent/images directory, +// for use with the next test. +func Cleanup(rke2TestLock int) error { + // Save the agent/images directory + cmd := exec.Command("cp", "-r", "/var/lib/rancher/rke2/agent/images", "/tmp/images-backup") + if res, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error backing up images directory: %s: %w", res, err) + } + + uninstall, err := findBundleExecutable("rke2-uninstall.sh") + if err != nil { + return err + } + // We don't care about the return value of the uninstall script, + // as it will always return an error because no rk2e service is running + exec.Command(uninstall).Run() + + // Restore the agent/images directory + if err := os.MkdirAll("/var/lib/rancher/rke2/agent", 0755); err != nil { + return fmt.Errorf("failed to make agent directory: %w", err) + } + cmd = exec.Command("mv", "/tmp/images-backup", "/var/lib/rancher/rke2/agent/images") + if res, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error restoring images directory: %s: %w", res, err) + } + if rke2TestLock != -1 { + return flock.Release(rke2TestLock) + } + return nil +} + +// ServerReady checks if the server is ready by checking the status of the pods +// and deployments that are required for the server to be operational +// On success, returns nil +func ServerReady() error { + hn, err := os.Hostname() + if err != nil { + return err + } + podsToCheck := []string{ + "etcd-" + hn, + "kube-apiserver-" + hn, + "kube-proxy-" + hn, + "kube-scheduler-" + hn, + } + deploymentsToCheck := []string{ + "rke2-coredns-rke2-coredns", + "rke2-metrics-server", + "rke2-snapshot-controller", + } + + pods, err := ParsePods("kube-system", metav1.ListOptions{}) + if err != nil { + return err + } + for _, pod := range podsToCheck { + ready := false + for _, p := range pods { + if p.Name == pod && p.Status.Phase == "Running" { + ready = true + } + } + if !ready { + return fmt.Errorf("pod %s is not ready", pod) + } + } + + return CheckDeployments(deploymentsToCheck) +} + +func ParsePods(namespace string, opts metav1.ListOptions) ([]corev1.Pod, error) { + clientSet, err := k8sClient() + if err != nil { + return nil, err + } + pods, err := clientSet.CoreV1().Pods(namespace).List(context.Background(), opts) + if err != nil { + return nil, err + } + + return pods.Items, nil +} + +// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error +func CheckDeployments(deployments []string) error { + + deploymentSet := make(map[string]bool) + for _, d := range deployments { + deploymentSet[d] = false + } + + client, err := k8sClient() + if err != nil { + return err + } + deploymentList, err := client.AppsV1().Deployments("kube-system").List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas { + deploymentSet[deployment.Name] = true + } + } + for d, found := range deploymentSet { + if !found { + return fmt.Errorf("deployment %s is not ready", d) + } + } + + return nil +} + +func contains(source []string, target string) bool { + for _, s := range source { + if s == target { + return true + } + } + return false +} + +// ServerArgsPresent checks if the given arguments are found in the running RKE2 server +func ServerArgsPresent(neededArgs []string) bool { + currentArgs, err := ServerArgs() + if err != nil { + logrus.Error(err) + return false + } + for _, arg := range neededArgs { + if !contains(currentArgs, arg) { + return false + } + } + return true +} + +// ServerArgs returns the list of arguments that the RKE2 server launched with +func ServerArgs() ([]string, error) { + results, err := RKE2Cmd("kubectl", "get", "nodes", "-o", `jsonpath='{.items[0].metadata.annotations.rke2\.io/node-args}'`) + if err != nil { + return nil, err + } + res := strings.ReplaceAll(results, "'", "") + var args []string + if err := json.Unmarshal([]byte(res), &args); err != nil { + return nil, err + } + return args, nil +} + +func k8sClient() (*kubernetes.Clientset, error) { + config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/rke2/rke2.yaml") + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return clientSet, nil +} diff --git a/tests/util/cmd.go b/tests/util/cmd.go deleted file mode 100644 index dd0fa64322..0000000000 --- a/tests/util/cmd.go +++ /dev/null @@ -1,101 +0,0 @@ -package util - -import ( - "encoding/json" - "os" - "os/exec" - "regexp" - "strings" - - "github.com/sirupsen/logrus" -) - -func findRke2Executable() string { - rke2Bin := "bin/rke2" - for { - _, err := os.Stat(rke2Bin) - if err != nil { - rke2Bin = "../" + rke2Bin - continue - } - break - } - return rke2Bin -} - -// Rke2Cmd launches the provided Rke2 command via exec. Command blocks until finished. -// Kubectl commands can also be passed. -// Command output from both Stderr and Stdout is provided via string. -// cmdEx1, err := Rke2Cmd("etcd-snapshot", "ls") -// cmdEx2, err := Rke2Cmd("kubectl", "get", "pods", "-A") -func Rke2Cmd(cmdName string, cmdArgs ...string) (string, error) { - if cmdName == "kubectl" { - byteOut, err := exec.Command(cmdName, cmdArgs...).CombinedOutput() - return string(byteOut), err - } - rke2Bin := findRke2Executable() - rke2Cmd := append([]string{cmdName}, cmdArgs...) - byteOut, err := exec.Command(rke2Bin, rke2Cmd...).CombinedOutput() - return string(byteOut), err -} - -func Rke2Ready() bool { - podsToCheck := []string{ - "etcd-rke2-server", - "kube-apiserver-rke2-server", - "kube-proxy-rke2-server", - "kube-scheduler-rke2-server", - "rke2-coredns-rke2-coredns", - } - pods, err := Rke2Cmd("kubectl", "get", "pods", "-A") - if err != nil { - return false - } - for _, pod := range podsToCheck { - reg := pod + ".+Running" - match, err := regexp.MatchString(reg, pods) - if !match || err != nil { - logrus.Error(err) - return false - } - } - return true -} - -func contains(source []string, target string) bool { - for _, s := range source { - if s == target { - return true - } - } - return false -} - -// ServerArgsPresent checks if the given arguments are found in the running k3s server -func ServerArgsPresent(neededArgs []string) bool { - currentArgs, err := Rke2ServerArgs() - if err != nil { - logrus.Error(err) - return false - } - for _, arg := range neededArgs { - if !contains(currentArgs, arg) { - return false - } - } - return true -} - -// Rke2ServerArgs returns the list of arguments that the rke2 server launched with -func Rke2ServerArgs() ([]string, error) { - results, err := Rke2Cmd("kubectl", "get", "nodes", "-o", `jsonpath='{.items[0].metadata.annotations.rke2\.io/node-args}'`) - if err != nil { - return nil, err - } - res := strings.ReplaceAll(results, "'", "") - var args []string - if err := json.Unmarshal([]byte(res), &args); err != nil { - return nil, err - } - return args, nil -}