CAA: add support to look up imagePullSecrets for pods

This PR initializes auth.json with the imagePullSecrets listed on the
pod and service account.

Fixes: #2231


Signed-off-by: Silenio Quarti <[email protected]>

src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml

@@ -45,10 +45,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-- name: auth-json-secret
-  namespace: confidential-containers-system
-  files:
-  #- auth.json # set - path to auth.json pull credentials file
 - name: peer-pods-secret
   namespace: confidential-containers-system
   # This file should look like this (w/o quotes!):

src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml

@@ -51,10 +51,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-- name: auth-json-secret
-  namespace: confidential-containers-system
-  files:
-  #- auth.json # set - path to auth.json pull credentials file
 - name: peer-pods-secret
   namespace: confidential-containers-system
   # This file should look like this (w/o quotes!):

src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml

@@ -36,10 +36,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-  - name: auth-json-secret
-    namespace: confidential-containers-system
-  #  files:
-  #- auth.json # set - path to auth.json pull credentials file
   - name: peer-pods-secret
     namespace: confidential-containers-system
     literals:

src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml

@@ -33,10 +33,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-- name: auth-json-secret
-  namespace: confidential-containers-system
-  files:
-  #- auth.json # set - path to auth.json pull credentials file
 - name: peer-pods-secret
   namespace: confidential-containers-system
   files:

src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml

@@ -44,10 +44,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-- name: auth-json-secret
-  namespace: confidential-containers-system
-  files:
-  #- auth.json # set - path to auth.json pull credentials file
 - name: peer-pods-secret
   namespace: confidential-containers-system
   # the below file should be in this format

src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml

@@ -42,10 +42,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-- name: auth-json-secret
-  namespace: confidential-containers-system
-  files:
-  #- auth.json # set - path to auth.json pull credentials file
 - name: peer-pods-secret
   namespace: confidential-containers-system
   literals:

src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml

@@ -40,10 +40,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-- name: auth-json-secret
-  namespace: confidential-containers-system
-  files:
-  #- auth.json # set - path to auth.json pull credentials file
 - name: ssh-key-secret
   namespace: confidential-containers-system
   files: # key generation example: ssh-keygen -f ./id_rsa -N "" && sudo cat id_rsa.pub >> /root/.ssh/authorized_keys

src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml

@@ -57,10 +57,6 @@ configMapGenerator:
 ##TLS_SETTINGS
 
 secretGenerator:
-- name: auth-json-secret
-  namespace: confidential-containers-system
-  files:
-  #- auth.json # set - path to auth.json pull credentials file
 - name: peer-pods-secret
   namespace: confidential-containers-system
   literals:

src/cloud-api-adaptor/install/rbac/peer-pod.yaml

@@ -14,6 +14,12 @@ rules:
 - apiGroups: [""]
   resources: ["pods"]
   verbs: ["get", "list"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+- apiGroups: [""]
+  resources: ["serviceaccounts"]
+  verbs: ["get", "list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

src/cloud-api-adaptor/install/yamls/caa-pod.yaml

@@ -48,9 +48,6 @@ spec:
           periodSeconds: 20
           initialDelaySeconds: 20
         volumeMounts:
-        - name: auth-json
-          mountPath: "/root/containers/" # hardcoded
-          readOnly: true
         - mountPath: /root/.ssh/
           name: ssh
           readOnly: true
@@ -73,10 +70,6 @@ spec:
       nodeSelector:
         node.kubernetes.io/worker: ""
       volumes:
-      - name: auth-json
-        secret:
-          secretName: auth-json-secret
-          optional: true # failing?
       - name: ssh
         secret:
           defaultMode: 384

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -36,8 +36,7 @@ import (
 )
 
 const (
-	SrcAuthfilePath = "/root/containers/auth.json"
-	Version         = "0.0.0"
+	Version = "0.0.0"
 )
 
 type InitData struct {
@@ -253,16 +252,13 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 
 	agentProxy := s.proxyFactory.New(serverName, socketPath)
 
-	var authJSON []byte
-	_, err = os.Stat(SrcAuthfilePath)
+	// Look up image pull secrets for the pod
+	authJSON, err := k8sops.GetImagePullSecrets(pod, namespace)
 	if err != nil {
-		logger.Printf("credential file %s is not present, skipping image auth config", SrcAuthfilePath)
-	} else {
-		authJSON, err = os.ReadFile(SrcAuthfilePath)
-		if err != nil {
-			return nil, fmt.Errorf("error reading %s: %v", SrcAuthfilePath, err)
-		}
-		logger.Printf("configure agent to use credentials file %s", SrcAuthfilePath)
+		return nil, fmt.Errorf("error reading image pull secrets: %w", err)
+	}
+	if authJSON != nil {
+		logger.Printf("successfully retrieved pod image pull secrets for %s/%s", namespace, pod)
 	}
 
 	daemonConfig := forwarder.Config{

src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go

@@ -81,6 +81,97 @@ func RemoveExtendedResources() error {
 	return nil
 }
 
+// Auths contains Registries with credentials
+type Auths struct {
+	Registries Registries `json:"auths"`
+}
+
+// Registries contains credentials for hosts
+type Registries map[string]Auth
+
+// Auth contains credentials for a given host
+type Auth struct {
+	Auth string `json:"auth"`
+}
+
+// GetImagePullSecrets gets image pull secrets for the specified pod
+func GetImagePullSecrets(podName string, namespace string) ([]byte, error) {
+
+	config, err := getKubeConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get k8s config: %v", err)
+	}
+
+	cli, err := getClient(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get k8s client: %v", err)
+	}
+
+	pod, err := cli.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	accountName := pod.Spec.ServiceAccountName
+	if accountName == "" {
+		accountName = "default"
+	}
+	serviceaAccount, err := cli.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), accountName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	auths := Auths{}
+	auths.Registries = make(map[string]Auth)
+	for _, secret := range serviceaAccount.ImagePullSecrets {
+		err := getAuths(cli, namespace, secret.Name, &auths)
+		if err != nil {
+			return nil, err
+		}
+	}
+	for _, secret := range pod.Spec.ImagePullSecrets {
+		err := getAuths(cli, namespace, secret.Name, &auths)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(auths.Registries) > 0 {
+		authJSON, err := json.Marshal(auths)
+		if err != nil {
+			return nil, err
+		}
+		return authJSON, nil
+	}
+	return nil, nil
+}
+
+// getAuths get auth credentials from specified docker secret
+func getAuths(cli *k8sclient.Clientset, namespace string, secretName string, auths *Auths) error {
+	secret, err := cli.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	registries := Registries{}
+	if secretData, ok := secret.Data[".dockerconfigjson"]; ok {
+		auths := Auths{}
+		err := json.Unmarshal(secretData, &auths)
+		if err != nil {
+			return err
+		}
+		registries = auths.Registries
+	} else if secretData, ok := secret.Data[".dockercfg"]; ok {
+		err = json.Unmarshal(secretData, &registries)
+		if err != nil {
+			return err
+		}
+	}
+	for registry, creds := range registries {
+		auths.Registries[registry] = creds
+	}
+	return nil
+}
+
 // patchNodeStatus patches the status of a node
 func patchNodeStatus(c *k8sclient.Clientset, nodeName string, patches []jsonPatch) error {
 	if len(patches) > 0 {

src/cloud-api-adaptor/test/e2e/assessment_runner.go

@@ -186,13 +186,16 @@ func writeAuthSecret(client klient.Client, ctx context.Context) error {
 		return nil
 	}
 
-	providerName := os.Getenv("CLOUD_PROVIDER")
-	// this path is relative to ./test/e2e
-	authFilePath := "../../install/overlays/" + providerName + "/auth.json"
-	authfile, err := os.ReadFile(authFilePath)
-	if err != nil {
-		return err
-	}
+	template := `{
+		"auths": {
+			"%s": {
+				"auth": "%s"
+			}
+		}
+	}`
+	cred := os.Getenv("REGISTRY_CREDENTIAL_ENCODED")
+	registryName := strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0]
+	authfile := []byte(fmt.Sprintf(template, registryName, cred))
 
 	secretData := map[string][]byte{v1.DockerConfigJsonKey: authfile}
 	secret = NewSecret(E2eNamespace, DEFAULT_AUTH_SECRET, secretData, v1.SecretTypeDockerConfigJson)

src/cloud-api-adaptor/test/provisioner/docker/provision_common.go

@@ -257,10 +257,6 @@ func (lio *DockerInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 		}
 	}
 
-	if err := lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil {
-		return err
-	}
-
 	if err := lio.Overlay.YamlReload(); err != nil {
 		return err
 	}

src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go

@@ -168,10 +168,6 @@ func (lio *IBMCloudInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config
 		}
 	}
 
-	if err = lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil {
-		return err
-	}
-
 	if err = lio.Overlay.YamlReload(); err != nil {
 		return err
 	}

src/cloud-api-adaptor/test/provisioner/kustomize.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"golang.org/x/exp/slices"
@@ -430,24 +429,3 @@ func setSecretGeneratorLiteral(k *ktypes.Kustomization, secretName string, key s
 
 	return nil
 }
-
-func (kh *KustomizeOverlay) SetAuthJsonSecretIfApplicable() error {
-	if cred := os.Getenv("REGISTRY_CREDENTIAL_ENCODED"); cred != "" {
-		registryName := strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0]
-		template := `{
-			"auths": {
-				"%s": {
-					"auth": "%s"
-				}
-			}
-		}`
-		authJSON := fmt.Sprintf(template, registryName, cred)
-		if err := os.WriteFile(filepath.Join(kh.ConfigDir, "auth.json"), []byte(authJSON), 0644); err != nil {
-			return err
-		}
-		if err := kh.SetKustomizeSecretGeneratorFile("auth-json-secret", "auth.json"); err != nil {
-			return err
-		}
-	}
-	return nil
-}

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -387,10 +387,6 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 		}
 	}
 
-	if err = lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil {
-		return err
-	}
-
 	if err = lio.Overlay.YamlReload(); err != nil {
 		return err
 	}