Merge pull request #971 from Azure/make-secrets-configurable

Make secrets configurable
Azure · Dec 13, 2024 · 34bf09c · 34bf09c
2 parents 4398a4e + 0a40e17
commit 34bf09c
Show file tree

Hide file tree

Showing 10 changed files with 99 additions and 58 deletions.
diff --git a/config/config.msft.yaml b/config/config.msft.yaml
@@ -85,6 +85,7 @@ defaults:
       enabled: true
       imageRepo: image-sync/component-sync
       repositories: quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package
+      secrets: ""
     ocMirror:
       enabled: true
       imageRepo: image-sync/oc-mirror

diff --git a/config/config.schema.json b/config/config.schema.json
@@ -236,14 +236,18 @@
               },
               "repositories": {
                 "type": "string"
+              },
+              "secrets": {
+                "type": "string"
               }
             },
             "additionalProperties": false,
             "required": [
               "enabled",
               "imageRepo",
               "imageTag",
-              "repositories"
+              "repositories",
+              "secrets"
             ]
           },
           "ocMirror": {

diff --git a/config/config.yaml b/config/config.yaml
@@ -86,6 +86,7 @@ defaults:
       imageRepo: image-sync/component-sync
       imageTag: latest
       repositories: quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package
+      secrets: '[{"registry": "quay.io", "secret": "bearer-secret"}]'
     ocMirror:
       enabled: true
       imageRepo: image-sync/oc-mirror

diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json
@@ -57,7 +57,8 @@
       "enabled": true,
       "imageRepo": "image-sync/component-sync",
       "imageTag": "latest",
-      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package"
+      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package",
+      "secrets": "[{\"registry\": \"quay.io\", \"secret\": \"bearer-secret\"}]"
     },
     "environmentName": "aro-hcp-image-sync",
     "keyVault": {

diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json
@@ -57,7 +57,8 @@
       "enabled": true,
       "imageRepo": "image-sync/component-sync",
       "imageTag": "latest",
-      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package"
+      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package",
+      "secrets": "[{\"registry\": \"quay.io\", \"secret\": \"bearer-secret\"}]"
     },
     "environmentName": "aro-hcp-image-sync",
     "keyVault": {

diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json
@@ -57,7 +57,8 @@
       "enabled": true,
       "imageRepo": "image-sync/component-sync",
       "imageTag": "0b3c08f",
-      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package"
+      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package",
+      "secrets": ""
     },
     "environmentName": "global-shared-resources",
     "keyVault": {

diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json
@@ -57,7 +57,8 @@
       "enabled": true,
       "imageRepo": "image-sync/component-sync",
       "imageTag": "latest",
-      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package"
+      "repositories": "quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package",
+      "secrets": "[{\"registry\": \"quay.io\", \"secret\": \"bearer-secret\"}]"
     },
     "environmentName": "aro-hcp-image-sync",
     "keyVault": {

diff --git a/dev-infrastructure/configurations/image-sync.tmpl.bicepparam b/dev-infrastructure/configurations/image-sync.tmpl.bicepparam
@@ -7,15 +7,17 @@ param keyVaultName = '{{ .imageSync.keyVault.name}}'
 param keyVaultPrivate = {{ .imageSync.keyVault.private }}
 param keyVaultSoftDelete = {{ .imageSync.keyVault.softDelete }}
 
-param bearerSecretName = 'bearer-secret'
 param componentSyncPullSecretName = 'component-sync-pull-secret'
 param componentSyncImage = '{{ .svcAcrName }}.azurecr.io/{{ .imageSync.componentSync.imageRepo }}:{{ .imageSync.componentSync.imageTag }}'
 param componentSyncEnabed = {{ .imageSync.componentSync.enabled }}
+param componentSyncSecrets = '{{ .imageSync.componentSync.secrets }}'
+
 param svcAcrName = '{{ .svcAcrName }}'
 
 param ocpAcrName = '{{ .ocpAcrName }}'
 param ocpPullSecretName = 'pull-secret'
 param repositoriesToSync = '{{ .imageSync.componentSync.repositories }}'
 param ocMirrorImage = '{{ .svcAcrName }}.azurecr.io/{{ .imageSync.ocMirror.imageRepo }}:{{ .imageSync.ocMirror.imageTag }}'
 param ocMirrorEnabled = {{ .imageSync.ocMirror.enabled }}
+
 param numberOfTags = 10
diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep
@@ -31,9 +31,6 @@ param keyVaultSoftDelete bool
 @description('The name of the pull secret for the component sync job')
 param componentSyncPullSecretName string
 
-@description('The name of the Quay API bearer token secret')
-param bearerSecretName string
-
 @description('The image to use for the component sync job')
 param componentSyncImage string
 
@@ -55,6 +52,22 @@ param ocMirrorEnabled bool
 @description('The name of the pull secret for the oc-mirror job')
 param ocpPullSecretName string
 
+@description('Secret configuration to pass into component sync')
+param componentSyncSecrets string
+
+var csSecrets = json(componentSyncSecrets)
+
+var bearerSecrets = [for css in csSecrets: '${css.secret}']
+
+var secretsFolder = '/etc/containers'
+
+var secretWithFolderPrefix = [
+  for css in csSecrets: {
+    registry: css.registry
+    secretFile: '${secretsFolder}/${css.secret}'
+  }
+]
+
 //
 // Container App Infra
 //
@@ -123,7 +136,7 @@ module acrPullRole '../modules/acr/acr-permissions.bicep' = {
 }
 
 module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = [
-  for secretName in [componentSyncPullSecretName, bearerSecretName, ocpPullSecretName]: {
+  for secretName in union([componentSyncPullSecretName, ocpPullSecretName], bearerSecrets): {
     name: guid(imageSyncManagedIdentity, location, keyVaultName, secretName, 'secret-user')
     params: {
       keyVaultName: keyVaultName
@@ -142,7 +155,31 @@ module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' =
 //
 
 var componentSyncJobName = 'component-sync'
-var pullSecretFile = 'quayio-auth.json'
+
+var componentSecretsArray = [
+  for bearerSecretName in bearerSecrets: {
+    name: 'bearer-secret'
+    keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${bearerSecretName}'
+    identity: uami.id
+  }
+]
+
+var componentSecretVolumesArray = [
+  for bearerSecretName in bearerSecrets: {
+    name: bearerSecretName
+    storageType: 'Secret'
+    secrets: [
+      { secretRef: bearerSecretName }
+    ]
+  }
+]
+
+var componentSecretVolumeMountsArray = [
+  for bearerSecretName in bearerSecrets: {
+    volumeName: bearerSecretName
+    mountPath: '/tmp/secrets/${bearerSecretName}'
+  }
+]
 
 resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = if (componentSyncEnabed) {
   name: componentSyncJobName
@@ -171,18 +208,16 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = if (componentSyncEna
           server: '${svcAcrName}${environment().suffixes.acrLoginServer}'
         }
       ]
-      secrets: [
-        {
-          name: 'pull-secrets'
-          keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${componentSyncPullSecretName}'
-          identity: uami.id
-        }
-        {
-          name: 'bearer-secret'
-          keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${bearerSecretName}'
-          identity: uami.id
-        }
-      ]
+      secrets: union(
+        [
+          {
+            name: 'pull-secrets'
+            keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${componentSyncPullSecretName}'
+            identity: uami.id
+          }
+        ],
+        componentSecretsArray
+      )
     }
     template: {
       containers: [
@@ -199,10 +234,7 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = if (componentSyncEna
             { name: 'TENANT_ID', value: tenant().tenantId }
             { name: 'DOCKER_CONFIG', value: '/auth' }
             { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId }
-            {
-              name: 'SECRETS'
-              value: '{"secrets":[{"registry": "quay.io", "secretfile": "/auth/${pullSecretFile}"}]}'
-            }
+            { name: 'SECRETS', value: string(secretWithFolderPrefix) }
           ]
         }
       ]
@@ -215,35 +247,33 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = if (componentSyncEna
           ]
           args: [
             '-c'
-            'cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cat /tmp/bearer-secret/bearer-secret | base64 -d > /etc/containers/${pullSecretFile}'
-          ]
-          volumeMounts: [
-            { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
-            { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' }
-            { volumeName: 'bearer-secret', mountPath: '/tmp/bearer-secret' }
-          ]
-        }
-      ]
-      volumes: [
-        {
-          name: 'pull-secrets-updated'
-          storageType: 'EmptyDir'
-        }
-        {
-          name: 'pull-secrets'
-          storageType: 'Secret'
-          secrets: [
-            { secretRef: 'pull-secrets' }
-          ]
-        }
-        {
-          name: 'bearer-secret'
-          storageType: 'Secret'
-          secrets: [
-            { secretRef: 'bearer-secret' }
+            'cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cd /tmp/secrets; for file in $(find . -type f); do; export fn=$(basename $file); cat $file | base64 -d > ${secretsFolder}/$fn; done;'
           ]
+          volumeMounts: union(
+            [
+              { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
+              { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' }
+            ],
+            componentSecretVolumeMountsArray
+          )
         }
       ]
+      volumes: union(
+        [
+          {
+            name: 'pull-secrets-updated'
+            storageType: 'EmptyDir'
+          }
+          {
+            name: 'pull-secrets'
+            storageType: 'Secret'
+            secrets: [
+              { secretRef: 'pull-secrets' }
+            ]
+          }
+        ],
+        componentSecretVolumesArray
+      )
     }
   }
   dependsOn: [

diff --git a/tooling/image-sync/internal/sync.go b/tooling/image-sync/internal/sync.go
@@ -31,9 +31,8 @@ type SyncConfig struct {
 	ManagedIdentityClientID string
 }
 type Secrets struct {
-	Registry        string
-	SecretFile      string
-	AzureSecretfile string
+	Registry   string
+	SecretFile string
 }
 
 // BearerSecret is the secret for the source OCI registry
@@ -148,9 +147,9 @@ func DoSync(cfg *SyncConfig) error {
 			if strings.HasSuffix(secret.Registry, "azurecr.io") ||
 				strings.HasSuffix(secret.Registry, "azurecr.cn") ||
 				strings.HasSuffix(secret.Registry, "azurecr.us") {
-				azureSecret, err := readAzureSecret(secret.AzureSecretfile)
+				azureSecret, err := readAzureSecret(secret.SecretFile)
 				if err != nil {
-					return fmt.Errorf("error reading azure secret file: %w %s", err, secret.AzureSecretfile)
+					return fmt.Errorf("error reading azure secret file: %w %s", err, secret.SecretFile)
 				}
 				bearerSecret, err := getACRBearerToken(ctx, *azureSecret, secret.Registry)
 				if err != nil {