feat: add checkpoint uds-core slim package (#818)

## Description This adds a ~75% faster way to deploy or reset a full uds-core cluster (theoretically would work for other preloaded things like testing GitLab Runner w/GitLab too). Normal: ![image](https://github.com/user-attachments/assets/a896e83e-1b9e-4da3-b5d7-022f493dd344) Checkpoint: ![image](https://github.com/user-attachments/assets/09b14230-33e0-4f33-b7d8-c6b85e882078) Tradeoffs: - Requires `sudo` - not sure of a great way around this without mangling volume permissions for containerd - May become unwieldy with more permutations (i.e. with layers work) - The cluster would be fully published (so all credentials are reused) ## Related Issue Fixes #N/A ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [X] New feature (non-breaking change which adds functionality) - [ ] Other (security config, docs update, etc) ## Checklist before merging - [ ] Test, docs, adr added or updated as needed - [X] [Contributor Guide](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md) followed --------- Co-authored-by: Micah Nagel <[email protected]>
defenseunicorns · Nov 18, 2024 · d95f6be · d95f6be
1 parent d4f85bb
commit d95f6be
Show file tree

Hide file tree

Showing 14 changed files with 340 additions and 0 deletions.
diff --git a/.github/workflows/checkpoint.yaml b/.github/workflows/checkpoint.yaml
@@ -0,0 +1,72 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+name: Checkpoint UDS Core
+
+on:
+  pull_request:
+    # milestoned is added here as a workaround for release-please not triggering PR workflows (PRs should be added to a milestone to trigger the workflow).
+    types: [milestoned, opened, reopened, synchronize]
+    paths:
+      - packages/checkpoint-dev/**
+      - .github/workflows/checkpoint**
+      - tasks/test.yaml
+      - "!**/*.md"
+      - "!**.jpg"
+      - "!**.png"
+      - "!**.gif"
+      - "!**.svg"
+  # triggered by tag-and-release.yaml
+  workflow_call:
+
+jobs:
+  checkpoint:
+    strategy:
+      matrix:
+        architecture: [amd64, arm64]
+    runs-on: ${{ matrix.architecture == 'arm64' && 'uds-ubuntu-arm64-4-core' || 'uds-ubuntu-big-boy-4-core' }}
+    name: UDS Core Checkpoint
+
+    permissions:
+      contents: read
+      packages: write
+      id-token: write  # This is needed for OIDC federation.
+
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Environment setup
+        uses: ./.github/actions/setup
+        with:
+          registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }}
+          registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }}
+          ghToken: ${{ secrets.GITHUB_TOKEN }}
+          chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }}
+
+      - name: Deploy K3d + UDS Core Slim Bundle
+        run: |
+          uds run -f tasks/deploy.yaml latest-slim-bundle-release --no-progress
+
+      - name: Create Checkpoint Package
+        run: |
+          uds run -f tasks/create.yaml checkpoint-dev-package --no-progress
+
+      - name: Test Checkpoint Package
+        run: |
+          uds run -f tasks/deploy.yaml checkpoint-package --no-progress
+          npm ci
+          uds run test:slim-dev --no-progress
+
+      - name: Debug Output
+        if: always()
+        uses: ./.github/actions/debug-output
+
+      - name: Publish Checkpoint Package
+        if: ${{ github.event_name == 'workflow_call' }}
+        run: uds run -f tasks/publish.yaml checkpoint-package --no-progress
+
+      - name: Save logs
+        if: always()
+        uses: ./.github/actions/save-logs
+        with:
+          suffix: -${{ matrix.architecture }}
diff --git a/.github/workflows/slim-dev-test.yaml b/.github/workflows/slim-dev-test.yaml
@@ -18,6 +18,7 @@ on:
       - packages/identity-authorization/**
       - bundles/k3d-slim-dev/**
       - .github/workflows/slim-dev**
+      - tasks/test.yaml
       - "!**/*.md"
       - "!**.jpg"
       - "!**.png"
@@ -52,6 +53,8 @@ jobs:
         uses: ./.github/actions/setup
       - name: Deploy Slim Dev Bundle
         run: uds run slim-dev --no-progress
+      - name: Test Slim Dev Bundle
+        run: uds run test:slim-dev --no-progress
       - name: Debug Output
         if: ${{ always() }}
         uses: ./.github/actions/debug-output

diff --git a/.github/workflows/tag-and-release.yaml b/.github/workflows/tag-and-release.yaml
@@ -32,3 +32,12 @@ jobs:
     with:
       snapshot: false
     secrets: inherit
+
+  checkpoint-uds-core-release:
+    needs: publish-uds-core-release
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+    uses: ./.github/workflows/checkpoint.yaml
+    secrets: inherit
diff --git a/.gitignore b/.gitignore
@@ -21,6 +21,9 @@ extract-terraform.sh
 **/.terraform*
 cluster-config.yaml
 **.tfstate
+
+*.tar
+
 **.backup
 **/.playwright/**
 **/.playwright
diff --git a/packages/base/tasks.yaml b/packages/base/tasks.yaml
@@ -7,6 +7,12 @@ includes:
 
 tasks:
   - name: validate
+    inputs:
+      validate_passthrough:
+        description: Whether to validate the passthrough gateway
+        default: "true"
     actions:
       - task: istio:validate
+        with:
+          validate_passthrough: ${{ .inputs.validate_passthrough }}
       - task: pepr:validate
diff --git a/packages/checkpoint-dev/README.md b/packages/checkpoint-dev/README.md
@@ -0,0 +1,34 @@
+# K3d + UDS Core Slim Dev Checkpoint
+
+This is a special Zarf package that takes a running K3d cluster (named `uds`) and wraps its committed container and volumes into a zarf package.
+
+> [!CAUTION]
+> This package does not currently work on macOS due to some limitations with filesystem permissions.
+
+## Creating this package
+
+In order to create this package you must follow the following:
+
+1. Setup a K3d cluster (named `uds`) containing the contents you'd like to checkpoint
+
+> [!NOTE]
+> The intent for this package is that those contents are the `uds dev stack`, `zarf init` and the `core-slim-dev` package (`core-base` and `core-identity-authorization`).
+
+2. Run `uds zarf package create <path-to-zarf-yaml> --confirm` on the Zarf Package in this directory
+
+> [!IMPORTANT]
+> This package requires `sudo` to create and deploy currently - if you see a prompt and it seems stalled it is waiting for password input (hidden by the spinner)
+
+## Deploying this package
+
+Once you have a package with the contents you want created you can deploy it with:
+
+```
+uds zarf package deploy <path-to-zarf-tarball> --confirm
+```
+
+> [!IMPORTANT]
+> This package requires `sudo` to deploy and create currently - if you see a prompt and it seems stalled it is waiting for password input (hidden by the spinner)
+
+> [!NOTE]
+> The pre-reqs for this package are the same as `uds-k3d` and you do not need to have a cluster running prior to deploying it.
diff --git a/packages/checkpoint-dev/checkpoint.sh b/packages/checkpoint-dev/checkpoint.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+# Name of the running k3d container
+K3S_CONTAINER="k3d-uds-server-0"
+
+if [ -z "$TMPDIR" ]; then
+    #  macOS sets TMPDIR to a user temp directory - this also provides more options to linux
+    TMPDIR="/tmp"
+fi
+DATA_DIR="${TMPDIR}/uds-checkpoint-data"
+
+# Step 0: Ensure we can get sudo
+echo "This package requires elevated permissions to create - requesting sudo (if paused enter password)"
+sudo echo "got sudo! success!"
+
+# Step 1: Get the container ID of the running k3d container
+CONTAINER_ID=$(docker ps -qf "name=$K3S_CONTAINER")
+
+if [ -z "$CONTAINER_ID" ]; then
+    echo "No running container found for $K3S_CONTAINER"
+    exit 1
+fi
+
+# Step 2: Get the mounted volumes of the running container
+echo "Inspecting container volumes for $CONTAINER_ID..."
+VOLUMES=$(docker inspect -f '{{ json .Mounts }}' "$CONTAINER_ID" | jq)
+
+# Step 3: Prepare directories to save the volume data
+sudo rm -rf "$DATA_DIR"
+mkdir -p "${DATA_DIR}/kubelet_data" "${DATA_DIR}/k3s_data"
+
+# Step 4: Loop through volumes and copy data to corresponding directories
+echo "Copying volumes to local directories..."
+
+for row in $(echo "$VOLUMES" | jq -r '.[] | @base64'); do
+    _jq() {
+        echo "${row}" | base64 --decode | jq -r "${1}"
+    }
+
+    SOURCE=$(_jq '.Source')
+    DESTINATION=$(_jq '.Destination')
+
+    case "$DESTINATION" in
+        "/var/lib/kubelet")
+            echo "Copying $SOURCE to ${DATA_DIR}/kubelet_data/"
+            sudo cp -a "$SOURCE"/. "${DATA_DIR}/kubelet_data/"
+            ;;
+        "/var/lib/rancher/k3s")
+            echo "Copying $SOURCE to ${DATA_DIR}/k3s_data/"
+            sudo cp -a "$SOURCE"/. "${DATA_DIR}/k3s_data/"
+            ;;
+        *)
+            echo "$DESTINATION is not needed. Skipping..."
+            ;;
+    esac
+done
+
+# Step 5: Commit and save the current container as a new image
+IMAGE_NAME="ghcr.io/defenseunicorns/uds-core/checkpoint:latest"
+echo "Committing container $CONTAINER_ID to image $IMAGE_NAME:latest..."
+docker commit -p "$CONTAINER_ID" "$IMAGE_NAME"
+
+echo "Saving image to ${DATA_DIR}/uds-k3d-checkpoint-latest.tar..."
+sudo docker save -o "${DATA_DIR}/uds-k3d-checkpoint-latest.tar" "$IMAGE_NAME"
+
+echo "Container image saved to ${DATA_DIR}/uds-k3d-checkpoint-latest.tar"
+
+# Step 6: Create a tarball from the data contents
+echo "Creating a final tarball to include in the package"
+sudo tar --blocking-factor=64 -cpf uds-checkpoint.tar -C "$DATA_DIR" .
+
+echo "Successfully checkpointed the cluster!"
+
+exit 0
diff --git a/packages/checkpoint-dev/zarf.yaml b/packages/checkpoint-dev/zarf.yaml
@@ -0,0 +1,97 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+kind: ZarfPackageConfig
+metadata:
+  name: k3d-core-slim-dev
+  description: "Rehydratable UDS K3d + UDS Core Slim (Istio, UDS Operator and Keycloak) Checkpoint"
+  authors: "Defense Unicorns - Product"
+  # x-release-please-start-version
+  version: "0.31.2"
+  # x-release-please-end
+
+variables:
+  - name: CLUSTER_NAME
+    description: "Name of the cluster"
+    default: "uds"
+
+  - name: K3D_EXTRA_ARGS
+    description: "Optionally pass k3d arguments to the default"
+    default: ""
+
+  - name: NGINX_EXTRA_PORTS
+    description: "Optionally allow more ports through Nginx (combine with K3D_EXTRA_ARGS '-p <port>:<port>@server:*')"
+    default: "[]"
+
+components:
+  - name: destroy-cluster
+    required: true
+    description: "Optionally destroy the cluster before creating it"
+    actions:
+      onDeploy:
+        before:
+          - cmd: |
+              echo "This package requires elevated permissions to deploy - requesting sudo (if paused enter password)"
+              sudo echo "got sudo! success!"
+          - cmd: k3d cluster delete ${ZARF_VAR_CLUSTER_NAME}
+            description: "Destroy the cluster"
+          - cmd: |
+              sudo rm -rf data
+
+  - name: create-cluster
+    required: true
+    description: "Create the K3d cluster w/UDS Core pre-installed"
+    files:
+      - source: uds-checkpoint.tar
+        target: uds-checkpoint.tar
+    actions:
+      onCreate:
+        before:
+          - cmd: ./checkpoint.sh
+        onSuccess:
+          - cmd: |
+              if [ -z "$TMPDIR" ]; then
+                #  macOS sets TMPDIR to a user temp directory - this also provides more options to linux
+                TMPDIR="/tmp"
+              fi
+              DATA_DIR="${TMPDIR}/uds-checkpoint-data"
+              sudo rm -rf "$DATA_DIR" uds-checkpoint.tar
+      onDeploy:
+        after:
+          - cmd: |
+              if [ -z "$TMPDIR" ]; then
+                #  macOS sets TMPDIR to a user temp directory - this also provides more options to linux
+                TMPDIR="/tmp"
+              fi
+              DATA_DIR="${TMPDIR}/uds-checkpoint-data"
+              mkdir -p "$DATA_DIR"
+
+              sudo tar --blocking-factor=64 -xpf uds-checkpoint.tar -C "$DATA_DIR"
+              K8S_TOKEN="$(sudo cat ${DATA_DIR}/k3s_data/server/token)"
+              echo $K8S_TOKEN
+              sudo docker load -i "${DATA_DIR}/uds-k3d-checkpoint-latest.tar"
+
+              k3d cluster create \
+                -p "80:80@server:*" \
+                -p "443:443@server:*" \
+                --api-port 6550 \
+                --k3s-arg "--disable=traefik@server:*" \
+                --k3s-arg "--disable=metrics-server@server:*" \
+                --k3s-arg "--disable=servicelb@server:*" \
+                --k3s-arg "--disable=local-storage@server:*" \
+                --k3s-arg "--token=${K8S_TOKEN}@server:*" \
+                -v "${DATA_DIR}/kubelet_data:/var/lib/kubelet@server:*" \
+                -v "${DATA_DIR}/k3s_data:/var/lib/rancher/k3s@server:*" \
+                --image ghcr.io/defenseunicorns/uds-core/checkpoint:latest ${ZARF_VAR_K3D_EXTRA_ARGS} \
+                ${ZARF_VAR_CLUSTER_NAME}
+            description: "Create the cluster"
+          # This action waits on Keycloak since it is the slowest pod to start after cluster creation. By waiting on it, we guarantee the cluster is healthy and usable after deployment.
+          - description: Keycloak to be Healthy
+            wait:
+              cluster:
+                kind: Pod
+                name: app.kubernetes.io/name=keycloak
+                namespace: keycloak
+                condition: Ready
+        onSuccess:
+          - cmd: rm -f uds-checkpoint.tar
diff --git a/release-please-config.json b/release-please-config.json
@@ -13,6 +13,7 @@
       "versioning": "default",
       "extra-files": [
         "README.md",
+        "packages/checkpoint-dev/zarf.yaml",
         "packages/base/zarf.yaml",
         "packages/identity-authorization/zarf.yaml",
         "packages/logging/zarf.yaml",

diff --git a/src/istio/tasks.yaml b/src/istio/tasks.yaml
@@ -3,6 +3,10 @@
 
 tasks:
   - name: validate
+    inputs:
+      validate_passthrough:
+        description: Whether to validate the passthrough gateway
+        default: "true"
     actions:
       - description: Validate the Istio Admin Gateway
         wait:
@@ -12,6 +16,7 @@ tasks:
             namespace: istio-admin-gateway
 
       - description: Validate the Istio Passthrough Gateway
+        if: ${{ eq .inputs.validate_passthrough "true" }}
         wait:
           cluster:
             kind: Gateway

diff --git a/tasks/create.yaml b/tasks/create.yaml
@@ -46,6 +46,12 @@ tasks:
       - description: "Create the slim dev bundle (Base and Identity)"
         cmd: "uds create bundles/k3d-slim-dev --confirm --no-progress --architecture=${ZARF_ARCHITECTURE}"
 
+  - name: checkpoint-dev-package
+    description: "Create the K3d + UDS Core Checkpoint Zarf Package"
+    actions:
+      - description: "Create the UDS Core Checkpoint Zarf Package"
+        cmd: "uds zarf package create packages/checkpoint-dev --confirm --no-progress --skip-sbom"
+
   # This task is a wrapper to support --set LAYER=identity-authorization
   - name: single-layer-callable
     actions:

diff --git a/tasks/deploy.yaml b/tasks/deploy.yaml
@@ -59,7 +59,17 @@ tasks:
       - description: "Deploy the latest UDS Core package release"
         cmd: uds zarf package deploy oci://${TARGET_REPO}/core:${LATEST_VERSION} --confirm --no-progress --components '*'
 
+  - name: latest-slim-bundle-release
+    actions:
+      - description: "Deploy the latest UDS Core package release"
+        cmd: uds deploy oci://ghcr.io/defenseunicorns/packages/uds/bundles/k3d-core-slim-dev:latest --set INSECURE_ADMIN_PASSWORD_GENERATION=true --confirm --no-progress
+
   - name: standard-package
     actions:
       - description: "Deploy the standard UDS Core zarf package"
         cmd: uds zarf package deploy build/zarf-package-core-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*'
+
+  - name: checkpoint-package
+    actions:
+      - description: "Deploy the checkpoint K3d + UDS Core Slim zarf package"
+        cmd: uds zarf package deploy build/zarf-package-k3d-core-slim-dev-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress