kuttl integration test

Run an e2e test with ephemeral kind cluster.
Usage: make test-e2e
The integration test is also executed as a github action

Signed-off-by: Leonardo Milleri <[email protected]>

.github/workflows/kuttl-int-tests.yaml

@@ -0,0 +1,33 @@
+name: "Integration tests"
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main 
+
+env:
+  KUTTL: /usr/local/bin/kubectl-kuttl
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-20.04
+    steps:
+
+    - uses: actions/[email protected]
+      with:
+        go-version: 1.21
+
+    - uses: actions/[email protected]
+
+    - name: Install dependencies
+      run: |
+        sudo curl -Lo $KUTTL https://github.com/kudobuilder/kuttl/releases/download/v0.19.0/kubectl-kuttl_0.19.0_linux_x86_64
+        sudo chmod +x $KUTTL
+        sudo curl -Lo kind https://github.com/kubernetes-sigs/kind/releases/download/v0.22.0/kind-linux-amd64
+        sudo chmod +x kind
+
+    - name: "Run integration tests"
+      run: make test-e2e

Makefile

@@ -172,6 +172,17 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default > dist/install.yaml
 
+# Run sample attestation in a kind cluster
+# pre-requirements: kuttl plugin and kind are installed
+# Usage: KBS_IMAGE_NAME=<trustee-image> CLIENT_IMAGE_NAME=<client-image> make test-e2e
+KBS_IMAGE_NAME ?= quay.io/confidential-containers/trustee:290fd0eb64ab20f50efbd27cf80542851c0ee17f
+CLIENT_IMAGE_NAME ?= quay.io/confidential-containers/kbs-client:v0.9.0
+.PHONY: test-e2e
+test-e2e:
+	./tests/scripts/kind-with-registry.sh
+	KBS_IMAGE_NAME=${KBS_IMAGE_NAME} CLIENT_IMAGE_NAME=${CLIENT_IMAGE_NAME} kubectl kuttl test || true
+	kind delete cluster
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to

README.md

@@ -312,6 +312,19 @@ which provide a reconcile function responsible for synchronizing resources until
 
 **NOTE:** You can also run this in one step by running: `make install run`
 
+### Integration tests
+
+An attestation with the sample-attester is performed in an ephemeral kind cluster
+Pre-requirements:
+- [kuttl](https://kuttl.dev/docs/cli.html#setup-the-kuttl-kubectl-plugin) plugin installed
+- [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) installed
+
+Optional: set the env variables KBS_IMAGE_NAME and CLIENT_IMAGE_NAME to override the default trustee/client images
+
+  ```sh
+  KBS_IMAGE_NAME=<trustee-image> CLIENT_IMAGE_NAME=<client-image> make test-e2e
+  ```
+
 ### Modifying the API definitions
 
 If you are editing the API definitions, generate the manifests such as CRs or CRDs using:

kuttl-test.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+- tests/e2e/
+timeout: 240

tests/e2e/sample-attester/00-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trustee-operator-controller-manager
+  namespace: trustee-operator-system
+status:
+  readyReplicas: 1

tests/e2e/sample-attester/00-install.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./install-operator.sh ${KBS_IMAGE_NAME} ${CLIENT_IMAGE_NAME}

tests/e2e/sample-attester/01-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kbs-auth-public-key
+  namespace: trustee-operator-system

tests/e2e/sample-attester/01-auth-secret.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./create-auth-secret.sh

tests/e2e/sample-attester/02-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kbsres1
+  namespace: trustee-operator-system

tests/e2e/sample-attester/02-kbs-secret.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./create-other-secret.sh

tests/e2e/sample-attester/03-kbs-config.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kbs-config
+  namespace: trustee-operator-system
+data:
+  kbs-config.json: |
+    {
+        "insecure_http" : true,
+        "sockets": ["0.0.0.0:8080"],
+        "auth_public_key": "/etc/auth-secret/publicKey",
+        "attestation_token_config": {
+          "attestation_token_type": "CoCo"
+        },
+        "repository_config": {
+          "type": "LocalFs",
+          "dir_path": "/opt/confidential-containers/kbs/repository"
+        },
+        "as_config": {
+          "work_dir": "/opt/confidential-containers/attestation-service",
+          "policy_engine": "opa",
+          "attestation_token_broker": "Simple",
+          "attestation_token_config": {
+            "duration_min": 5
+          },
+          "rvps_config": {
+            "store_type": "LocalJson",
+            "store_config": {
+              "file_path": "/opt/confidential-containers/rvps/reference-values/reference-values.json"
+            }
+          }
+        },
+        "policy_engine_config": {
+          "policy_path": "/opt/confidential-containers/opa/policy.rego"
+        }
+    }

tests/e2e/sample-attester/04-resource-policy.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: resource-policy
+  namespace: trustee-operator-system
+data:
+  policy.rego: |
+    package policy
+
+    default allow = false
+    allow {
+            input["tcb-status"]["sample.svn"] == "1"
+    }

tests/e2e/sample-attester/05-rvps-reference-values.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rvps-reference-values
+  namespace: trustee-operator-system
+data:
+  reference-values.json: |
+    [
+    ]

tests/e2e/sample-attester/06-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trustee-deployment
+  namespace: trustee-operator-system
+status:
+  readyReplicas: 1

tests/e2e/sample-attester/06-kbsconfig_sample.yaml

@@ -0,0 +1,21 @@
+apiVersion: confidentialcontainers.org/v1alpha1
+kind: KbsConfig
+metadata:
+  labels:
+    app.kubernetes.io/name: kbsconfig
+    app.kubernetes.io/instance: kbsconfig-sample
+    app.kubernetes.io/part-of: trustee-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: trustee-operator
+  name: kbsconfig-sample
+  namespace: trustee-operator-system
+spec:
+  kbsConfigMapName: kbs-config
+  kbsAuthSecretName: kbs-auth-public-key
+  kbsDeploymentType: AllInOneDeployment
+  kbsRvpsRefValuesConfigMapName: rvps-reference-values
+  kbsResourcePolicyConfigMapName: resource-policy
+  kbsSecretResources:
+    - "kbsres1"
+  KbsEnvVars: 
+    RUST_LOG: debug  

tests/e2e/sample-attester/07-assert.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kbs-client
+  namespace: trustee-operator-system
+status:
+  containerStatuses:
+  - ready: true

tests/e2e/sample-attester/07-client.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl apply -k .
+

tests/e2e/sample-attester/08-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trustee-secret
+  namespace: trustee-operator-system

tests/e2e/sample-attester/08-get-secret.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./get-secret.sh

tests/e2e/sample-attester/client.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kbs-client
+  namespace: trustee-operator-system
+spec:
+  containers:
+  - name: kbs-client
+    image: quay.io/confidential-containers/kbs-client:v0.9.0
+    imagePullPolicy: IfNotPresent
+    command:
+      - sleep
+      - "360000"
+    env:
+      - name: RUST_LOG
+        value:  none

tests/e2e/sample-attester/create-auth-secret.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+openssl genpkey -algorithm ed25519 > privateKey
+openssl pkey -in privateKey -pubout -out publicKey
+kubectl create secret generic kbs-auth-public-key --from-file=publicKey -n trustee-operator-system

tests/e2e/sample-attester/create-other-secret.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kubectl create secret generic kbsres1 --from-literal key1=res1val1 --from-literal key2=res1val2 -n trustee-operator-system

tests/e2e/sample-attester/get-secret.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+SECRET="$(kubectl exec -it -n trustee-operator-system kbs-client -- kbs-client --url http://kbs-service:8080 get-resource --path default/kbsres1/key1)"
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error when retrieving the secret"
+else
+    echo "Attestation completed successfully: secret="$SECRET
+    # this secret is created only to check the secret retrieval has been successful
+    kubectl create secret generic trustee-secret --from-literal key1=$SECRET -n trustee-operator-system
+fi

tests/e2e/sample-attester/install-operator.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+KBS_IMAGE_NAME=$1
+CLIENT_IMAGE_NAME=$2
+REGISTRY=localhost:5001
+export IMG=${REGISTRY}/trustee-operator:test
+
+# project root dir
+pushd ../../..
+
+pushd config/manager
+kustomize edit set image controller=$IMG
+kustomize edit add patch --patch "- op: replace
+  path: '/spec/template/spec/containers/0/env/1'
+  value:
+    name: KBS_IMAGE_NAME
+    value: ${KBS_IMAGE_NAME}" --kind Deployment --name controller-manager
+popd
+make docker-build docker-push
+make build-installer
+kubectl apply -f dist/install.yaml
+
+pushd tests/e2e/sample-attester
+kustomize edit set image $CLIENT_IMAGE_NAME
+popd

tests/e2e/sample-attester/kustomization.yaml

@@ -0,0 +1,3 @@
+# There resources need to be edited with kustomize
+resources:
+- client.yaml

tests/scripts/kind-with-registry.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+set -o errexit
+
+# 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
+    registry:2
+fi
+
+# 2. Create kind cluster with containerd registry config dir enabled
+# TODO: kind will eventually enable this by default and this patch will
+# be unnecessary.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+EOF
+
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes); do
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF