e2e-test: update kbs test case to support kbs with ibmse verifier

- deploy kbs with ibmse verifer base on ENV IBM_SE_CREDS_DIR
- update kbs test case for libvrit provider to support kbs with ibmse verifier
- set key source before test cases
- `deny_all.rego` and `allow_with_wrong_image_tag.rego` are FAIL cases
- `allow_with_correct_claims.rego` is PASS case

fixes confidential-containers#1934

Signed-off-by: Da Li Liu <[email protected]>

src/cloud-api-adaptor/test/e2e/common.go

@@ -24,7 +24,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const CURL_IMAGE = "quay.io/curl/curl:latest"
 const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest"
 const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180
 const DEFAULT_AUTH_SECRET = "auth-json-secret-default"
@@ -39,6 +38,10 @@ func isTestWithTrusteeOperator() bool {
 	return os.Getenv("TEST_TRUSTEE_OPERATOR") == "yes"
 }
 
+func isTestWithKbsIBMSE() bool {
+	return os.Getenv("IBM_SE_CREDS_DIR") != ""
+}
+
 func isTestOnCrio() bool {
 	return os.Getenv("CONTAINER_RUNTIME") == "crio"
 }
@@ -187,10 +190,6 @@ func NewPodWithInitContainer(namespace string, podName string) *corev1.Pod {
 	return NewPod(namespace, podName, "busybox", BUSYBOX_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"}), WithInitContainers(initContainer))
 }
 
-func NewCurlPodWithName(namespace, podName string) *corev1.Pod {
-	return NewPod(namespace, podName, "curl", CURL_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
-}
-
 func NewBusyboxPodWithName(namespace, podName string) *corev1.Pod {
 	return NewPod(namespace, podName, "busybox", BUSYBOX_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
 }

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -585,7 +585,7 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) {
 			ContainerName: pod.Spec.Containers[0].Name,
 			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
 				if strings.Contains(stdout.String(), "This is my cluster name") {
-					log.Infof("Success to get key.bin %s", stdout.String())
+					log.Infof("Success to get key.bin: %s", stdout.String())
 					return true
 				} else {
 					log.Errorf("Failed to access key.bin: %s", stdout.String())
@@ -603,21 +603,28 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) {
 func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) {
 
 	log.Info("Do test kbs key release failure case")
-	pod := NewCurlPodWithName(E2eNamespace, "curl-failure")
+	pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget-failure")
 	testCommands := []TestCommand{
 		{
-			Command:       []string{"curl", "-s", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"},
+			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"},
 			ContainerName: pod.Spec.Containers[0].Name,
-			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
-				body := stdout.String()
-				if strings.Contains(strings.ToLower(body), "error") {
-					log.Infof("Pass failure case as: %s", stdout.String())
+			TestErrorFn: func(err error) bool {
+				if strings.Contains(err.Error(), "command terminated with exit code 1") {
 					return true
 				} else {
-					log.Errorf("Failed to faliure case as: %s", stdout.String())
+					log.Errorf("Got unexpected error: %s", err.Error())
 					return false
 				}
 			},
+			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
+				if strings.Contains(stdout.String(), "This is my cluster name") {
+					log.Errorf("FAIL as successed to get key.bin: %s", stdout.String())
+					return false
+				} else {
+					log.Infof("PASS as failed to access key.bin: %s", stdout.String())
+					return true
+				}
+			},
 		},
 	}
 

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -108,12 +108,25 @@ func TestLibvirtKbsKeyRelease(t *testing.T) {
 	if !isTestWithKbs() {
 		t.Skip("Skipping kbs related test as kbs is not deployed")
 	}
-	_ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego")
+	_ = keyBrokerService.SetSampleSecretKey()
+	_ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
+	_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego")
 	assert := LibvirtAssert{}
 	t.Parallel()
 	DoTestKbsKeyReleaseForFailure(t, testEnv, assert)
-	_ = keyBrokerService.EnableKbsCustomizedPolicy("allow_all.rego")
-	DoTestKbsKeyRelease(t, testEnv, assert)
+	if isTestWithKbsIBMSE() {
+		t.Log("KBS with ibmse cases")
+		// the allow_*_.rego file is created by follow document
+		// https://github.com/confidential-containers/trustee/blob/main/deps/verifier/src/se/README.md#set-attestation-policy
+		_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego")
+		DoTestKbsKeyReleaseForFailure(t, testEnv, assert)
+		_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego")
+		DoTestKbsKeyRelease(t, testEnv, assert)
+	} else {
+		t.Log("KBS normal cases")
+		_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego")
+		DoTestKbsKeyRelease(t, testEnv, assert)
+	}
 }
 
 func TestLibvirtRestrictivePolicyBlocksExec(t *testing.T) {

src/cloud-api-adaptor/test/e2e/main_test.go

@@ -155,7 +155,7 @@ func TestMain(m *testing.M) {
 		var kbsparams string
 		if shouldDeployKbs {
 			log.Info("Deploying kbs")
-			if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil {
+			if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg); err != nil {
 				return ctx, err
 			}
 

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -102,7 +102,7 @@ func getHardwarePlatform() (string, error) {
 	return strings.TrimSuffix(string(out), "\n"), err
 }
 
-func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
+func NewKeyBrokerService(clusterName string, cfg *envconf.Config) (*KeyBrokerService, error) {
 	log.Info("creating key.bin")
 
 	// Create secret
@@ -135,7 +135,6 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		log.Errorf("%v", err)
 		return nil, err
 	}
-	fmt.Println(k8sCnfDir)
 
 	kbsCert := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.pem")
 	if _, err := os.Stat(kbsCert); os.IsNotExist(err) {
@@ -197,6 +196,31 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 
 	}
 
+	// IBM_SE_CREDS_DIR describe at https://github.com/confidential-containers/trustee/blob/main/kbs/config/kubernetes/README.md#deploy-kbs
+	ibmseCredsDir := os.Getenv("IBM_SE_CREDS_DIR")
+	if ibmseCredsDir != "" {
+		log.Info("IBM_SE_CREDS_DIR is providered, deploy KBS with IBM SE verifier")
+		// We always deploy the KBS pod to first worker node
+		workerNodeIP, workerNodeName, _ := getFirstWorkerNodeIPAndName(cfg)
+		log.Infof("Copying IBM_SE_CREDS files to first worker node: %s", workerNodeIP)
+		err := copyGivenFilesToWorkerNode(ibmseCredsDir, workerNodeIP)
+		if err != nil {
+			return nil, err
+		}
+		log.Infof("Creating PV for kbs with ibmse")
+		pvFilePath := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/s390x/pv.yaml")
+		err = createPVonTargetWorkerNode(pvFilePath, workerNodeName, cfg)
+		if err != nil {
+			return nil, err
+		}
+		patchFile := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/s390x/patch.yaml")
+		// skip the SE related certs check as we are running the test case on a dev machine
+		err = skipSeCertsVerification(patchFile)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	overlay, err := NewBaseKbsInstallOverlay(TRUSTEE_REPO_PATH)
 	if err != nil {
 		return nil, err
@@ -208,6 +232,122 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 	}, nil
 }
 
+func skipSeCertsVerification(patchFile string) error {
+	data, err := os.ReadFile(patchFile)
+	if err != nil {
+		return fmt.Errorf("failed to read file: %v", err)
+	}
+	content := string(data)
+	content = strings.Replace(content, "false", "true", -1)
+	err = os.WriteFile(patchFile, []byte(content), 0644)
+	if err != nil {
+		return fmt.Errorf("failed to write file: %v", err)
+	}
+	return nil
+}
+
+func createPVonTargetWorkerNode(pvFilePath, nodeName string, cfg *envconf.Config) error {
+	data, err := os.ReadFile(pvFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to read file: %v", err)
+	}
+	content := string(data)
+	content = strings.Replace(content, "${IBM_SE_CREDS_DIR}", "/root/ibmse", -1)
+	content = strings.Replace(content, "${NODE_NAME}", nodeName, -1)
+	err = os.WriteFile(pvFilePath, []byte(content), 0644)
+	if err != nil {
+		return fmt.Errorf("failed to write file: %v", err)
+	}
+
+	cmd := exec.Command("kubectl", "apply", "-f", pvFilePath)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
+	stdoutStderr, err := cmd.CombinedOutput()
+	log.Tracef("%v, output: %s", cmd, stdoutStderr)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getFirstWorkerNodeIPAndName(cfg *envconf.Config) (string, string, error) {
+	client, err := cfg.NewClient()
+	if err != nil {
+		return "", "", err
+	}
+	nodeList := &corev1.NodeList{}
+	if err := client.Resources("").List(context.TODO(), nodeList); err != nil {
+		return "", "", err
+	}
+	// Filter out control plane nodes and get the IP of the first worker node
+	for _, node := range nodeList.Items {
+		if isWorkerNode(&node) {
+			return node.Status.Addresses[0].Address, node.Name, nil
+		}
+	}
+	return "", "", fmt.Errorf("no worker nodes found")
+}
+
+func isWorkerNode(node *corev1.Node) bool {
+	// Check for the existence of the label or taint that identifies control plane nodes
+	_, isMaster := node.Labels["node-role.kubernetes.io/master"]
+	_, isControlPlane := node.Labels["node-role.kubernetes.io/control-plane"]
+	if isMaster || isControlPlane {
+		return false
+	}
+	return true
+}
+
+func copyGivenFilesToWorkerNode(sourceDir, targetNodeIP string) error {
+	// Step 1: Compress the source directory using tar
+	tarFilePath, err := compressDirectory(sourceDir)
+	if err != nil {
+		return fmt.Errorf("failed to compress directory: %v", err)
+	}
+	defer os.Remove(tarFilePath) // Clean up the temporary tar file
+
+	// Step 2: Transfer the compressed file to the target node using SCP
+	targetFilePath := "/tmp/" + filepath.Base(tarFilePath)
+	err = transferFile(tarFilePath, targetNodeIP, targetFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to transfer file: %v", err)
+	}
+
+	// Step 3: Decompress the file on the target node
+	err = decompressFileOnTargetNode(targetNodeIP, targetFilePath, "/root")
+	if err != nil {
+		return fmt.Errorf("failed to decompress file on target node: %v", err)
+	}
+
+	return nil
+}
+
+func compressDirectory(sourceDir string) (string, error) {
+	tarFilePath := sourceDir + ".tar.gz"
+	cmd := exec.Command("tar", "-czf", tarFilePath, "-C", filepath.Dir(sourceDir), filepath.Base(sourceDir))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+	return tarFilePath, nil
+}
+
+func transferFile(localFilePath, targetNodeIP, remoteFilePath string) error {
+	cmd := exec.Command("scp", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", localFilePath, fmt.Sprintf("root@%s:%s", targetNodeIP, remoteFilePath))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func decompressFileOnTargetNode(targetNodeIP, remoteFilePath, targetDir string) error {
+	cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", fmt.Sprintf("root@%s", targetNodeIP), fmt.Sprintf("tar -xzf %s -C %s", remoteFilePath, targetDir))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
 func NewCloudAPIAdaptor(provider string, installDir string) (*CloudAPIAdaptor, error) {
 	namespace := "confidential-containers-system"
 
@@ -408,7 +548,7 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf
 	return "", fmt.Errorf("Service %s not found", serviceName)
 }
 
-func (p *KeyBrokerService) EnableKbsCustomizedPolicy(customizedOpaFile string) error {
+func (p *KeyBrokerService) EnableKbsCustomizedResourcePolicy(customizedOpaFile string) error {
 	kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release")
 	privateKey := "../../kbs/config/kubernetes/base/kbs.key"
 	policyFile := filepath.Join("../../kbs/sample_policies", customizedOpaFile)
@@ -424,6 +564,42 @@ func (p *KeyBrokerService) EnableKbsCustomizedPolicy(customizedOpaFile string) e
 	return nil
 }
 
+func (p *KeyBrokerService) EnableKbsCustomizedAttestationPolicy(customizedOpaFile string) error {
+	kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release")
+	privateKey := "../../kbs/config/kubernetes/base/kbs.key"
+	policyFile := filepath.Join("../../kbs/sample_policies", customizedOpaFile)
+	log.Info("EnableKbsCustomizedPolicy: ", policyFile)
+	cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-attestation-policy", "--policy-file", policyFile)
+	cmd.Dir = kbsClientDir
+	cmd.Env = os.Environ()
+	stdoutStderr, err := cmd.CombinedOutput()
+	log.Tracef("%v, output: %s", cmd, stdoutStderr)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (p *KeyBrokerService) SetSampleSecretKey() error {
+	kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release")
+	privateKey := "../../kbs/config/kubernetes/base/kbs.key"
+	platform, err := getHardwarePlatform()
+	if err != nil {
+		return err
+	}
+	keyFilePath := "../../kbs/config/kubernetes/overlays/" + platform + "/key.bin"
+	log.Info("set key resource: ", keyFilePath)
+	cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-resource", "--path", "reponame/workload_key/key.bin", "--resource-file", keyFilePath)
+	cmd.Dir = kbsClientDir
+	cmd.Env = os.Environ()
+	stdoutStderr, err := cmd.CombinedOutput()
+	log.Tracef("%v, output: %s", cmd, stdoutStderr)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error {
 	log.Info("Customize the overlay yaml file")
 	if err := p.installOverlay.Edit(ctx, cfg, props); err != nil {

src/cloud-api-adaptor/test/tools/provisioner-cli/main.go

@@ -94,7 +94,7 @@ func main() {
 			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
 				log.Fatal("kbs image not provided")
 			}
-			keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"])
+			keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg)
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -178,7 +178,7 @@ func main() {
 
 		if shouldDeployKbs {
 			props := provisioner.GetProperties(context.TODO(), cfg)
-			keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"])
+			keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg)
 			if err != nil {
 				log.Fatal(err)
 			}