diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index c56be4562..c5780aae8 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -24,7 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const CURL_IMAGE = "quay.io/curl/curl:latest" const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest" const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180 const DEFAULT_AUTH_SECRET = "auth-json-secret-default" @@ -39,6 +38,10 @@ func isTestWithTrusteeOperator() bool { return os.Getenv("TEST_TRUSTEE_OPERATOR") == "yes" } +func isTestWithKbsIBMSE() bool { + return os.Getenv("IBM_SE_CREDS_DIR") != "" +} + func isTestOnCrio() bool { return os.Getenv("CONTAINER_RUNTIME") == "crio" } @@ -187,10 +190,6 @@ func NewPodWithInitContainer(namespace string, podName string) *corev1.Pod { return NewPod(namespace, podName, "busybox", BUSYBOX_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"}), WithInitContainers(initContainer)) } -func NewCurlPodWithName(namespace, podName string) *corev1.Pod { - return NewPod(namespace, podName, "curl", CURL_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"})) -} - func NewBusyboxPodWithName(namespace, podName string) *corev1.Pod { return NewPod(namespace, podName, "busybox", BUSYBOX_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"})) } diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 4c411830f..871e79b5a 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -585,7 +585,7 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { ContainerName: pod.Spec.Containers[0].Name, TestCommandStdoutFn: func(stdout bytes.Buffer) bool { if strings.Contains(stdout.String(), "This is my cluster name") { - log.Infof("Success to get key.bin %s", stdout.String()) + log.Infof("Success to get key.bin: %s", stdout.String()) return true } else { log.Errorf("Failed to access key.bin: %s", stdout.String()) @@ -603,21 +603,28 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) { log.Info("Do test kbs key release failure case") - pod := NewCurlPodWithName(E2eNamespace, "curl-failure") + pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget-failure") testCommands := []TestCommand{ { - Command: []string{"curl", "-s", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"}, + Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"}, ContainerName: pod.Spec.Containers[0].Name, - TestCommandStdoutFn: func(stdout bytes.Buffer) bool { - body := stdout.String() - if strings.Contains(strings.ToLower(body), "error") { - log.Infof("Pass failure case as: %s", stdout.String()) + TestErrorFn: func(err error) bool { + if strings.Contains(err.Error(), "command terminated with exit code 1") { return true } else { - log.Errorf("Failed to faliure case as: %s", stdout.String()) + log.Errorf("Got unexpected error: %s", err.Error()) return false } }, + TestCommandStdoutFn: func(stdout bytes.Buffer) bool { + if strings.Contains(stdout.String(), "This is my cluster name") { + log.Errorf("FAIL as successed to get key.bin: %s", stdout.String()) + return false + } else { + log.Infof("PASS as failed to access key.bin: %s", stdout.String()) + return true + } + }, }, } diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 422ee5eda..52b37e81a 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -108,12 +108,25 @@ func TestLibvirtKbsKeyRelease(t *testing.T) { if !isTestWithKbs() { t.Skip("Skipping kbs related test as kbs is not deployed") } - _ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego") + _ = keyBrokerService.SetSampleSecretKey() + _ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") + _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego") assert := LibvirtAssert{} t.Parallel() DoTestKbsKeyReleaseForFailure(t, testEnv, assert) - _ = keyBrokerService.EnableKbsCustomizedPolicy("allow_all.rego") - DoTestKbsKeyRelease(t, testEnv, assert) + if isTestWithKbsIBMSE() { + t.Log("KBS with ibmse cases") + // the allow_*_.rego file is created by follow document + // https://github.com/confidential-containers/trustee/blob/main/deps/verifier/src/se/README.md#set-attestation-policy + _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego") + DoTestKbsKeyReleaseForFailure(t, testEnv, assert) + _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego") + DoTestKbsKeyRelease(t, testEnv, assert) + } else { + t.Log("KBS normal cases") + _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") + DoTestKbsKeyRelease(t, testEnv, assert) + } } func TestLibvirtRestrictivePolicyBlocksExec(t *testing.T) { diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 87b428383..fef5fff8a 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -155,7 +155,7 @@ func TestMain(m *testing.M) { var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") - if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil { + if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg); err != nil { return ctx, err } diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index b3ebd0575..7001c2b4b 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -5,15 +5,10 @@ package provisioner import ( "context" - "crypto/ed25519" - "crypto/rand" - "crypto/x509" - "encoding/pem" "fmt" "os" "os/exec" "path/filepath" - "strings" "time" "github.com/BurntSushi/toml" @@ -85,129 +80,6 @@ type InstallOverlay interface { // Waiting timeout for bringing up the pod const PodWaitTimeout = time.Second * 30 -// trustee repo related base path -const TRUSTEE_REPO_PATH = "../trustee" - -func saveToFile(filename string, content []byte) error { - // Save contents to file - err := os.WriteFile(filename, content, 0644) - if err != nil { - return fmt.Errorf("writing contents to file: %w", err) - } - return nil -} - -func getHardwarePlatform() (string, error) { - out, err := exec.Command("uname", "-i").Output() - return strings.TrimSuffix(string(out), "\n"), err -} - -func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { - log.Info("creating key.bin") - - // Create secret - content := []byte("This is my cluster name: " + clusterName) - platform, err := getHardwarePlatform() - if err != nil { - return nil, err - } - filePath := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/"+platform+"/key.bin") - // Create the file. - file, err := os.Create(filePath) - if err != nil { - err = fmt.Errorf("creating file: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - defer file.Close() - - // Write the content to the file. - err = saveToFile(filePath, content) - if err != nil { - err = fmt.Errorf("writing to the file: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - - k8sCnfDir, err := os.Getwd() - if err != nil { - err = fmt.Errorf("getting the current working directory: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - fmt.Println(k8sCnfDir) - - kbsCert := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.pem") - if _, err := os.Stat(kbsCert); os.IsNotExist(err) { - kbsKey := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.key") - keyOutputFile, err := os.Create(kbsKey) - if err != nil { - err = fmt.Errorf("creating key file: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - defer keyOutputFile.Close() - - pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - - b, err := x509.MarshalPKCS8PrivateKey(privateKey) - if err != nil { - err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - - privateKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "PRIVATE KEY", - Bytes: b, - }) - - // Save private key to file - err = saveToFile(kbsKey, privateKeyPEM) - if err != nil { - err = fmt.Errorf("saving private key to file: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - - b, err = x509.MarshalPKIXPublicKey(pubKey) - if err != nil { - err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - - publicKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "PUBLIC KEY", - Bytes: b, - }) - - // Save public key to file - err = saveToFile(kbsCert, publicKeyPEM) - if err != nil { - err = fmt.Errorf("saving public key to file: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - - } - - overlay, err := NewBaseKbsInstallOverlay(TRUSTEE_REPO_PATH) - if err != nil { - return nil, err - } - - return &KeyBrokerService{ - installOverlay: overlay, - endpoint: "", - }, nil -} - func NewCloudAPIAdaptor(provider string, installDir string) (*CloudAPIAdaptor, error) { namespace := "confidential-containers-system" @@ -272,191 +144,6 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro return overlayFunc(installDir, provider) } -func NewBaseKbsInstallOverlay(installDir string) (InstallOverlay, error) { - log.Info("Creating kbs install overlay") - overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base/")) - if err != nil { - return nil, err - } - - return &KbsInstallOverlay{ - overlay: overlay, - }, nil -} - -func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { - log.Info("Creating kbs install overlay") - platform, err := getHardwarePlatform() - if err != nil { - return nil, err - } - overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/nodeport/"+platform)) - if err != nil { - return nil, err - } - - return &KbsInstallOverlay{ - overlay: overlay, - }, nil -} - -func (lio *KbsInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error { - return lio.overlay.Apply(ctx, cfg) -} - -func (lio *KbsInstallOverlay) Delete(ctx context.Context, cfg *envconf.Config) error { - return lio.overlay.Delete(ctx, cfg) -} - -func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, props map[string]string) error { - var err error - log.Infof("Updating kbs image with %q", props["KBS_IMAGE"]) - if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newName", props["KBS_IMAGE"]); err != nil { - return err - } - - log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"]) - if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil { - return err - } - - return nil -} - -func getNodeIPForSvc(deploymentName string, service corev1.Service, cfg *envconf.Config) (string, error) { - client, err := cfg.NewClient() - if err != nil { - return "", err - } - podList := &corev1.PodList{} - if err := client.Resources(service.Namespace).List(context.TODO(), podList); err != nil { - return "", err - } - - nodeList := &corev1.NodeList{} - if err := client.Resources("").List(context.TODO(), nodeList); err != nil { - return "", err - } - - var matchingPod *corev1.Pod - for i := range podList.Items { - pod := &podList.Items[i] - if pod.Labels["app"] == deploymentName { - matchingPod = pod - break - } - } - - for _, node := range nodeList.Items { - if node.Name == matchingPod.Spec.NodeName { - return node.Status.Addresses[0].Address, nil - } - } - - return "", fmt.Errorf("Node IP not found for Service %s", service.Name) -} - -func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Config) (string, error) { - client, err := cfg.NewClient() - if err != nil { - return "", err - } - - namespace := "coco-tenant" - serviceName := "kbs" - deploymentName := "kbs" - - resources := client.Resources(namespace) - - kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}} - fmt.Printf("Wait for the %s deployment be available\n", deploymentName) - if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), - wait.WithTimeout(time.Minute*2)); err != nil { - return "", err - } - - services := &corev1.ServiceList{} - if err := resources.List(context.TODO(), services); err != nil { - return "", err - } - - for _, service := range services.Items { - if service.ObjectMeta.Name == serviceName { - // Ensure the service is of type NodePort - if service.Spec.Type != corev1.ServiceTypeNodePort { - return "", fmt.Errorf("Service %s is not of type NodePort", "kbs") - } - - var nodePort int32 - // Extract NodePort - if len(service.Spec.Ports) > 0 { - nodePort = service.Spec.Ports[0].NodePort - } else { - return "", fmt.Errorf("NodePort is not configured for Service %s", "kbs") - } - - nodeIP, err := getNodeIPForSvc(deploymentName, service, cfg) - if err != nil { - return "", err - } - - p.endpoint = fmt.Sprintf("http://%s:%d", nodeIP, nodePort) - return p.endpoint, nil - } - } - - return "", fmt.Errorf("Service %s not found", serviceName) -} - -func (p *KeyBrokerService) EnableKbsCustomizedPolicy(customizedOpaFile string) error { - kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release") - privateKey := "../../kbs/config/kubernetes/base/kbs.key" - policyFile := filepath.Join("../../kbs/sample_policies", customizedOpaFile) - log.Info("EnableKbsCustomizedPolicy: ", policyFile) - cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", policyFile) - cmd.Dir = kbsClientDir - cmd.Env = os.Environ() - stdoutStderr, err := cmd.CombinedOutput() - log.Tracef("%v, output: %s", cmd, stdoutStderr) - if err != nil { - return err - } - return nil -} - -func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { - log.Info("Customize the overlay yaml file") - if err := p.installOverlay.Edit(ctx, cfg, props); err != nil { - return err - } - - // Create kustomize pointer for overlay directory with updated changes - tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH) - if err != nil { - return err - } - - log.Info("Install Kbs") - if err := tmpoverlay.Apply(ctx, cfg); err != nil { - return err - } - return nil -} - -func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error { - // Create kustomize pointer for overlay directory with updated changes - tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH) - if err != nil { - return err - } - - log.Info("Uninstall the cloud-api-adaptor") - if err = tmpoverlay.Delete(ctx, cfg); err != nil { - return err - } - return nil -} - // Deletes the peer pods installation including the controller manager. func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error { client, err := cfg.NewClient() diff --git a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go new file mode 100644 index 000000000..b0e8024eb --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go @@ -0,0 +1,510 @@ +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package provisioner + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + log "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" +) + +// trustee repo related base path +const TRUSTEE_REPO_PATH = "../trustee" + +func getHardwarePlatform() (string, error) { + out, err := exec.Command("uname", "-i").Output() + return strings.TrimSuffix(string(out), "\n"), err +} + +func NewKeyBrokerService(clusterName string, cfg *envconf.Config) (*KeyBrokerService, error) { + log.Info("creating key.bin") + + // Create secret + content := []byte("This is my cluster name: " + clusterName) + platform, err := getHardwarePlatform() + if err != nil { + return nil, err + } + filePath := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/"+platform+"/key.bin") + // Create the file. + file, err := os.Create(filePath) + if err != nil { + err = fmt.Errorf("creating file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + defer file.Close() + + // Write the content to the file. + err = saveToFile(filePath, content) + if err != nil { + err = fmt.Errorf("writing to the file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + k8sCnfDir, err := os.Getwd() + if err != nil { + err = fmt.Errorf("getting the current working directory: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + kbsCert := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.pem") + if _, err := os.Stat(kbsCert); os.IsNotExist(err) { + kbsKey := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.key") + keyOutputFile, err := os.Create(kbsKey) + if err != nil { + err = fmt.Errorf("creating key file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + defer keyOutputFile.Close() + + pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + b, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: b, + }) + + // Save private key to file + err = saveToFile(kbsKey, privateKeyPEM) + if err != nil { + err = fmt.Errorf("saving private key to file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + b, err = x509.MarshalPKIXPublicKey(pubKey) + if err != nil { + err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + publicKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Bytes: b, + }) + + // Save public key to file + err = saveToFile(kbsCert, publicKeyPEM) + if err != nil { + err = fmt.Errorf("saving public key to file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + } + + // IBM_SE_CREDS_DIR describe at https://github.com/confidential-containers/trustee/blob/main/kbs/config/kubernetes/README.md#deploy-kbs + ibmseCredsDir := os.Getenv("IBM_SE_CREDS_DIR") + if ibmseCredsDir != "" { + log.Info("IBM_SE_CREDS_DIR is providered, deploy KBS with IBM SE verifier") + // We always deploy the KBS pod to first worker node + workerNodeIP, workerNodeName, _ := getFirstWorkerNodeIPAndName(cfg) + log.Infof("Copying IBM_SE_CREDS files to first worker node: %s", workerNodeIP) + err := copyGivenFilesToWorkerNode(ibmseCredsDir, workerNodeIP) + if err != nil { + return nil, err + } + log.Infof("Creating PV for kbs with ibmse") + pvFilePath := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/s390x/pv.yaml") + err = createPVonTargetWorkerNode(pvFilePath, workerNodeName, cfg) + if err != nil { + return nil, err + } + patchFile := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/s390x/patch.yaml") + // skip the SE related certs check as we are running the test case on a dev machine + err = skipSeCertsVerification(patchFile) + if err != nil { + return nil, err + } + } + + overlay, err := NewBaseKbsInstallOverlay(TRUSTEE_REPO_PATH) + if err != nil { + return nil, err + } + + return &KeyBrokerService{ + installOverlay: overlay, + endpoint: "", + }, nil +} + +func saveToFile(filename string, content []byte) error { + // Save contents to file + err := os.WriteFile(filename, content, 0644) + if err != nil { + return fmt.Errorf("writing contents to file: %w", err) + } + return nil +} + +func skipSeCertsVerification(patchFile string) error { + data, err := os.ReadFile(patchFile) + if err != nil { + return fmt.Errorf("failed to read file: %v", err) + } + content := string(data) + content = strings.Replace(content, "false", "true", -1) + err = os.WriteFile(patchFile, []byte(content), 0644) + if err != nil { + return fmt.Errorf("failed to write file: %v", err) + } + return nil +} + +func createPVonTargetWorkerNode(pvFilePath, nodeName string, cfg *envconf.Config) error { + data, err := os.ReadFile(pvFilePath) + if err != nil { + return fmt.Errorf("failed to read file: %v", err) + } + content := string(data) + content = strings.Replace(content, "${IBM_SE_CREDS_DIR}", "/root/ibmse", -1) + content = strings.Replace(content, "${NODE_NAME}", nodeName, -1) + err = os.WriteFile(pvFilePath, []byte(content), 0644) + if err != nil { + return fmt.Errorf("failed to write file: %v", err) + } + + cmd := exec.Command("kubectl", "apply", "-f", pvFilePath) + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + + return nil +} + +func getFirstWorkerNodeIPAndName(cfg *envconf.Config) (string, string, error) { + client, err := cfg.NewClient() + if err != nil { + return "", "", err + } + nodeList := &corev1.NodeList{} + if err := client.Resources("").List(context.TODO(), nodeList); err != nil { + return "", "", err + } + // Filter out control plane nodes and get the IP of the first worker node + for _, node := range nodeList.Items { + if isWorkerNode(&node) { + return node.Status.Addresses[0].Address, node.Name, nil + } + } + return "", "", fmt.Errorf("no worker nodes found") +} + +func isWorkerNode(node *corev1.Node) bool { + // Check for the existence of the label or taint that identifies control plane nodes + _, isMaster := node.Labels["node-role.kubernetes.io/master"] + _, isControlPlane := node.Labels["node-role.kubernetes.io/control-plane"] + if isMaster || isControlPlane { + return false + } + return true +} + +func copyGivenFilesToWorkerNode(sourceDir, targetNodeIP string) error { + // Step 1: Compress the source directory using tar + tarFilePath, err := compressDirectory(sourceDir) + if err != nil { + return fmt.Errorf("failed to compress directory: %v", err) + } + defer os.Remove(tarFilePath) // Clean up the temporary tar file + + // Step 2: Transfer the compressed file to the target node using SCP + targetFilePath := "/tmp/" + filepath.Base(tarFilePath) + err = transferFile(tarFilePath, targetNodeIP, targetFilePath) + if err != nil { + return fmt.Errorf("failed to transfer file: %v", err) + } + + // Step 3: Decompress the file on the target node + err = decompressFileOnTargetNode(targetNodeIP, targetFilePath, "/root") + if err != nil { + return fmt.Errorf("failed to decompress file on target node: %v", err) + } + + return nil +} + +func compressDirectory(sourceDir string) (string, error) { + tarFilePath := sourceDir + ".tar.gz" + cmd := exec.Command("tar", "-czf", tarFilePath, "-C", filepath.Dir(sourceDir), filepath.Base(sourceDir)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + return "", err + } + return tarFilePath, nil +} + +func transferFile(localFilePath, targetNodeIP, remoteFilePath string) error { + cmd := exec.Command("scp", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", localFilePath, fmt.Sprintf("root@%s:%s", targetNodeIP, remoteFilePath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func decompressFileOnTargetNode(targetNodeIP, remoteFilePath, targetDir string) error { + cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", fmt.Sprintf("root@%s", targetNodeIP), fmt.Sprintf("tar -xzf %s -C %s", remoteFilePath, targetDir)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func NewBaseKbsInstallOverlay(installDir string) (InstallOverlay, error) { + log.Info("Creating kbs install overlay") + overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base/")) + if err != nil { + return nil, err + } + + return &KbsInstallOverlay{ + overlay: overlay, + }, nil +} + +func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { + log.Info("Creating kbs install overlay") + platform, err := getHardwarePlatform() + if err != nil { + return nil, err + } + overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/nodeport/"+platform)) + if err != nil { + return nil, err + } + + return &KbsInstallOverlay{ + overlay: overlay, + }, nil +} + +func (lio *KbsInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error { + return lio.overlay.Apply(ctx, cfg) +} + +func (lio *KbsInstallOverlay) Delete(ctx context.Context, cfg *envconf.Config) error { + return lio.overlay.Delete(ctx, cfg) +} + +func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, props map[string]string) error { + var err error + log.Infof("Updating kbs image with %q", props["KBS_IMAGE"]) + if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newName", props["KBS_IMAGE"]); err != nil { + return err + } + + log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"]) + if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil { + return err + } + + return nil +} + +func getNodeIPForSvc(deploymentName string, service corev1.Service, cfg *envconf.Config) (string, error) { + client, err := cfg.NewClient() + if err != nil { + return "", err + } + podList := &corev1.PodList{} + if err := client.Resources(service.Namespace).List(context.TODO(), podList); err != nil { + return "", err + } + + nodeList := &corev1.NodeList{} + if err := client.Resources("").List(context.TODO(), nodeList); err != nil { + return "", err + } + + var matchingPod *corev1.Pod + for i := range podList.Items { + pod := &podList.Items[i] + if pod.Labels["app"] == deploymentName { + matchingPod = pod + break + } + } + + for _, node := range nodeList.Items { + if node.Name == matchingPod.Spec.NodeName { + return node.Status.Addresses[0].Address, nil + } + } + + return "", fmt.Errorf("Node IP not found for Service %s", service.Name) +} + +func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Config) (string, error) { + client, err := cfg.NewClient() + if err != nil { + return "", err + } + + namespace := "coco-tenant" + serviceName := "kbs" + deploymentName := "kbs" + + resources := client.Resources(namespace) + + kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}} + fmt.Printf("Wait for the %s deployment be available\n", deploymentName) + if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(time.Minute*2)); err != nil { + return "", err + } + + services := &corev1.ServiceList{} + if err := resources.List(context.TODO(), services); err != nil { + return "", err + } + + for _, service := range services.Items { + if service.ObjectMeta.Name == serviceName { + // Ensure the service is of type NodePort + if service.Spec.Type != corev1.ServiceTypeNodePort { + return "", fmt.Errorf("Service %s is not of type NodePort", "kbs") + } + + var nodePort int32 + // Extract NodePort + if len(service.Spec.Ports) > 0 { + nodePort = service.Spec.Ports[0].NodePort + } else { + return "", fmt.Errorf("NodePort is not configured for Service %s", "kbs") + } + + nodeIP, err := getNodeIPForSvc(deploymentName, service, cfg) + if err != nil { + return "", err + } + + p.endpoint = fmt.Sprintf("http://%s:%d", nodeIP, nodePort) + return p.endpoint, nil + } + } + + return "", fmt.Errorf("Service %s not found", serviceName) +} + +func (p *KeyBrokerService) EnableKbsCustomizedResourcePolicy(customizedOpaFile string) error { + kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release") + privateKey := "../../kbs/config/kubernetes/base/kbs.key" + policyFile := filepath.Join("../../kbs/sample_policies", customizedOpaFile) + log.Info("EnableKbsCustomizedPolicy: ", policyFile) + cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", policyFile) + cmd.Dir = kbsClientDir + cmd.Env = os.Environ() + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + return nil +} + +func (p *KeyBrokerService) EnableKbsCustomizedAttestationPolicy(customizedOpaFile string) error { + kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release") + privateKey := "../../kbs/config/kubernetes/base/kbs.key" + policyFile := filepath.Join("../../kbs/sample_policies", customizedOpaFile) + log.Info("EnableKbsCustomizedPolicy: ", policyFile) + cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-attestation-policy", "--policy-file", policyFile) + cmd.Dir = kbsClientDir + cmd.Env = os.Environ() + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + return nil +} + +func (p *KeyBrokerService) SetSampleSecretKey() error { + kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release") + privateKey := "../../kbs/config/kubernetes/base/kbs.key" + platform, err := getHardwarePlatform() + if err != nil { + return err + } + keyFilePath := "../../kbs/config/kubernetes/overlays/" + platform + "/key.bin" + log.Info("set key resource: ", keyFilePath) + cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-resource", "--path", "reponame/workload_key/key.bin", "--resource-file", keyFilePath) + cmd.Dir = kbsClientDir + cmd.Env = os.Environ() + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + return nil +} + +func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { + log.Info("Customize the overlay yaml file") + if err := p.installOverlay.Edit(ctx, cfg, props); err != nil { + return err + } + + // Create kustomize pointer for overlay directory with updated changes + tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH) + if err != nil { + return err + } + + log.Info("Install Kbs") + if err := tmpoverlay.Apply(ctx, cfg); err != nil { + return err + } + return nil +} + +func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error { + // Create kustomize pointer for overlay directory with updated changes + tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH) + if err != nil { + return err + } + + log.Info("Uninstall the cloud-api-adaptor") + if err = tmpoverlay.Delete(ctx, cfg); err != nil { + return err + } + return nil +} diff --git a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go index 4f862b1bf..c97836a4e 100644 --- a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go +++ b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go @@ -94,7 +94,7 @@ func main() { if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { log.Fatal("kbs image not provided") } - keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg) if err != nil { log.Fatal(err) } @@ -178,7 +178,7 @@ func main() { if shouldDeployKbs { props := provisioner.GetProperties(context.TODO(), cfg) - keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg) if err != nil { log.Fatal(err) }