Provisioner: Add kbs provisioning steps in CI

Fixes: #1676
Signed-off-by: Kartik Joshi <[email protected]>

.github/workflows/azure-e2e-test.yml

@@ -25,6 +25,12 @@ on:
       caa-image:
         type: string
         description: prebuilt caa image
+      kbs-image-id:
+        type: string
+        description: prebuild kbs image
+      kbs-image-tag:
+        type: string
+        description: prebuild kbs image tag
 
 jobs:
   generate-podvm-image-version:
@@ -116,10 +122,17 @@ jobs:
       with:
         go-version: ${{ env.GO_VERSION }}
 
+    - name: Extract provisioner environment
+      run: |
+        echo "KBS_IMAGE=$(yq -e '.oci.kbs.registry' versions.yaml)" >> "$GITHUB_ENV"
+        echo "KBS_IMAGE_TAG=$(yq -e '.oci.kbs.tag' versions.yaml)" >> "$GITHUB_ENV"
+
     - name: Create provisioner file
       env:
         AZURE_IMAGE_ID: ${{ github.event.inputs.podvm-image-id || format('/CommunityGalleries/{0}/images/{1}/Versions/{2}', vars.AZURE_COMMUNITY_GALLERY_NAME, vars.AZURE_PODVM_IMAGE_DEF_NAME, needs.generate-podvm-image-version.outputs.image-version) }}
         CAA_IMAGE: "${{ github.event.inputs.caa-image || needs.build-caa-container-image.outputs.caa-image }}"
+        KBS_IMAGE: ${{ github.event.inputs.kbs-image-id || env.KBS_IMAGE }}
+        KBS_IMAGE_TAG: ${{  github.event.inputs.kbs-image-tag || env.KBS_IMAGE_TAG}}
       run: |
         cat << EOF > "$TEST_PROVISION_FILE"
           AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
@@ -132,6 +145,8 @@ jobs:
           IS_CI_MANAGED_CLUSTER="true"
           MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}"
           CAA_IMAGE="${CAA_IMAGE}"
+          KBS_IMAGE="${KBS_IMAGE}"
+          KBS_IMAGE_TAG="${KBS_IMAGE_TAG}"
         EOF
         cat "$TEST_PROVISION_FILE"
         # assert that no variable is unset
@@ -203,9 +218,20 @@ jobs:
         test -n "$SUBNET_ID"
         echo "AZURE_SUBNET_ID=\"${SUBNET_ID}\"" >> "$TEST_PROVISION_FILE"
 
+    - name: Extract kbs reference
+      run: echo "KBS_VERSION=$(yq -e '.git.kbs.reference' versions.yaml)" >> "$GITHUB_ENV"
+
+    - name: Checkout kbs Repository
+      run: |
+        git clone https://github.com/confidential-containers/trustee test/trustee
+        pushd test/trustee
+        git checkout "${KBS_VERSION}"
+        popd
+
     - name: Run e2e test
       env:
         TEST_PROVISION: "no"
+        DEPLOY_KBS: "yes"
       run: |
         # Since we install the cluster in parallel, we need to get the credentials here.
         az aks get-credentials \

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -80,6 +80,9 @@ type InstallOverlay interface {
 // Waiting timeout for bringing up the pod
 const PodWaitTimeout = time.Second * 30
 
+// trustee repo related base path
+const TRUSTEE_REPO_PATH = "../trustee"
+
 func saveToFile(filename string, content []byte) error {
 	// Save contents to file
 	err := os.WriteFile(filename, content, 0644)
@@ -94,7 +97,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 
 	// Create secret
 	content := []byte("This is my cluster name: " + clusterName)
-	filePath := "trustee/kbs/config/kubernetes/overlays/key.bin"
+	filePath := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/key.bin")
 	// Create the file.
 	file, err := os.Create(filePath)
 	if err != nil {
@@ -120,9 +123,9 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 	}
 	fmt.Println(k8sCnfDir)
 
-	kbsCert := filepath.Join(k8sCnfDir, "trustee/kbs/config/kubernetes/base/kbs.pem")
+	kbsCert := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.pem")
 	if _, err := os.Stat(kbsCert); os.IsNotExist(err) {
-		kbsKey := filepath.Join(k8sCnfDir, "trustee/kbs/config/kubernetes/base/kbs.key")
+		kbsKey := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.key")
 		keyOutputFile, err := os.Create(kbsKey)
 		if err != nil {
 			err = fmt.Errorf("creating key file: %w\n", err)
@@ -174,7 +177,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 
 	}
 
-	overlay, err := NewBaseKbsInstallOverlay("trustee")
+	overlay, err := NewBaseKbsInstallOverlay(TRUSTEE_REPO_PATH)
 	if err != nil {
 		return nil, err
 	}
@@ -372,7 +375,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop
 	}
 
 	// Create kustomize pointer for overlay directory with updated changes
-	tmpoverlay, err := NewKbsInstallOverlay("trustee")
+	tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH)
 	if err != nil {
 		return err
 	}
@@ -386,7 +389,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop
 
 func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error {
 	// Create kustomize pointer for overlay directory with updated changes
-	tmpoverlay, err := NewKbsInstallOverlay("trustee")
+	tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH)
 	if err != nil {
 		return err
 	}

src/cloud-api-adaptor/versions.yaml

@@ -41,6 +41,9 @@ git:
   opa:
     url: https://github.com/open-policy-agent/opa
     reference: v0.58.0
+  kbs:
+    url: https://github.com/confidential-containers/trustee
+    reference: 84432a2a97da306399db5bc863c9324dbd8b95ac
 oci:
   pause:
     registry: docker://registry.k8s.io/pause