Merge pull request #718 from Azure/deploy-sync-as-jobs

Use containerapps for component-sync
Azure · Oct 25, 2024 · 971240f · 971240f
2 parents da11385 + e9917e0
commit 971240f
Show file tree

Hide file tree

Showing 16 changed files with 272 additions and 35 deletions.
diff --git a/dev-infrastructure/configurations/dev.mk b/dev-infrastructure/configurations/dev.mk
@@ -4,6 +4,6 @@ REGIONAL_RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION)
 SVC_KV_RESOURCEGROUP ?= global
 GLOBAL_RESOURCEGROUP ?= global
 IMAGE_SYNC_RESOURCEGROUP ?= aro-hcp-image-sync-$(USER)-$(REGION)
-IMAGE_SYNC_ENVIRONMENT ?= image-sync-env
+IMAGE_SYNC_ENVIRONMENT ?= image-sync-env-sxo4oqbcjiekg
 ARO_HCP_IMAGE_ACR ?= arohcpdev
 REPOSITORIES_TO_SYNC ?= '{registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/pstefans/controlplaneoperator,quay.io/app-sre/uhc-clusters-service}'
diff --git a/dev-infrastructure/configurations/mvp-image-sync.bicepparam b/dev-infrastructure/configurations/mvp-image-sync.bicepparam
@@ -1,10 +1,10 @@
 using '../templates/image-sync.bicep'
 
-param acrResourceGroup = 'gobal'
+param acrResourceGroup = 'global'
 
 param keyVaultName = 'aro-hcp-dev-global-kv'
 
 param requiredSecretNames = [
-  'pull-secret'
+  'component-sync-pull-secret'
   'bearer-secret'
 ]
diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep
@@ -53,13 +53,23 @@ resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
 
 module acrContributorRole '../modules/acr-permissions.bicep' = {
   name: guid(imageSyncManagedIdentity, 'acr', 'readwrite')
+  scope: resourceGroup(acrResourceGroup)
   params: {
     principalId: uami.properties.principalId
     grantPushAccess: true
     acrResourceGroupid: acrResourceGroup
   }
 }
 
+module acrPullRole '../modules/acr-permissions.bicep' = {
+  name: guid(imageSyncManagedIdentity, 'acr', 'pull')
+  scope: resourceGroup(acrResourceGroup)
+  params: {
+    principalId: uami.properties.principalId
+    acrResourceGroupid: acrResourceGroup
+  }
+}
+
 module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = [
   for secretName in requiredSecretNames: {
     name: '${secretName}-access'

diff --git a/image-sync/configuration/mvp-image-sync.yml b/image-sync/configuration/mvp-image-sync.yml
@@ -0,0 +1,9 @@
+repositories:
+  - registry.k8s.io/external-dns/external-dns
+  - quay.io/acm-d/rhtap-hypershift-operator
+  - quay.io/pstefans/controlplaneoperator
+  - quay.io/app-sre/uhc-clusters-service
+numberOfTags: 10
+quaySecretfile: /auth/quayio-auth.json
+acrRegistry: arohcpdev.azurecr.io
+tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408
diff --git a/image-sync/configuration/mvp-oc-mirror.yml b/image-sync/configuration/mvp-oc-mirror.yml
@@ -0,0 +1,30 @@
+kind: ImageSetConfiguration
+apiVersion: mirror.openshift.io/v1alpha2
+storageConfig:
+  registry:
+    imageURL: arohcpdev.azurecr.io/mirror/oc-mirror-metadata
+    skipTLS: false
+mirror:
+  platform:
+    architectures:
+      - multi
+      - amd64
+    channels:
+      - name: stable-4.16
+        minVersion: 4.16.0
+        maxVersion: 4.16.3
+        type: ocp
+      - name: stable-4.17
+        minVersion: 4.17.0
+        maxVersion: 4.17.0
+        type: ocp
+    graph: true
+  additionalImages:
+  - name: registry.redhat.io/redhat/redhat-operator-index:v4.16
+  - name: registry.redhat.io/redhat/certified-operator-index:v4.16
+  - name: registry.redhat.io/redhat/community-operator-index:v4.16
+  - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.16
+  - name: registry.redhat.io/redhat/redhat-operator-index:v4.17
+  - name: registry.redhat.io/redhat/certified-operator-index:v4.17
+  - name: registry.redhat.io/redhat/community-operator-index:v4.17
+  - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17
diff --git a/image-sync/deployment/Makefile b/image-sync/deployment/Makefile
@@ -14,24 +14,24 @@ deploy-shared:
 	--set serviceAccountName=image-sync \
 	--set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID}
 
-deploy-component-sync:
-	AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \
-			-g ${RESOURCEGROUP} \
-			-n image-sync \
-			--query clientId) && \
-	TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
-	helm upgrade --force --install component-sync-cronjob ./componentSyncCronjob \
-	--namespace ${NAMESPACE} --create-namespace \
-	--set serviceAccountName=image-sync \
-	--set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID} \
-	--set acrRegistryName=${ARO_HCP_IMAGE_ACR} \
-	--set azureTenantId=$${TENANT_ID} \
-	--set componentSyncImage="arohcpdev.azurecr.io/image-sync/component-sync" \
-	--set componentSyncTag=latest \
-	--set credsBearerSecret=bearer-secret \
-	--set credsPullSecret=component-pull-secret \
-	--set credsKeyVaultName=service-kv-aro-hcp-dev \
-	--set repositories=${REPOSITORIES_TO_SYNC}
+# deploy-component-sync:
+# 	AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \
+# 			-g ${RESOURCEGROUP} \
+# 			-n image-sync \
+# 			--query clientId) && \
+# 	TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
+# 	helm upgrade --force --install component-sync-cronjob ./componentSyncCronjob \
+# 	--namespace ${NAMESPACE} --create-namespace \
+# 	--set serviceAccountName=image-sync \
+# 	--set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID} \
+# 	--set acrRegistryName=${ARO_HCP_IMAGE_ACR} \
+# 	--set azureTenantId=$${TENANT_ID} \
+# 	--set componentSyncImage="arohcpdev.azurecr.io/image-sync/component-sync" \
+# 	--set componentSyncTag=latest \
+# 	--set credsBearerSecret=bearer-secret \
+# 	--set credsPullSecret=component-pull-secret \
+# 	--set credsKeyVaultName=service-kv-aro-hcp-dev \
+# 	--set repositories=${REPOSITORIES_TO_SYNC}
 
 deploy-oc-mirror:
 	AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \
@@ -63,4 +63,11 @@ undeploy-oc-mirror:
 
 undeploy: undeploy-shared undeploy-component-sync undeploy-oc-mirror
 
+deploy-component-sync:
+	az deployment group create --name comp-sync \
+		--resource-group ${IMAGE_SYNC_RESOURCEGROUP} \
+		--template-file componentSync/component-sync.bicep \
+		--parameters componentSync/mvp-component-sync.bicepparam \
+		--parameters containerImage="arohcpdev.azurecr.io/image-sync/component-sync:latest"
+
 .PHONY: deploy-component-sync deploy-shared deploy-oc-mirror undeploy-shared undeploy-component-sync undeploy-oc-mirror
diff --git a/image-sync/deployment/componentSync/component-sync.bicep b/image-sync/deployment/componentSync/component-sync.bicep
@@ -0,0 +1,123 @@
+@description('Azure Region Location')
+param location string = resourceGroup().location
+
+@description('Name of the Container App Environment')
+param environmentName string
+
+@description('Name of the Container App Job')
+param jobName string
+
+@description('Container image to use for the job')
+param containerImage string
+
+@description('Name of the user assigned managed identity')
+param imageSyncManagedIdentity string
+
+@description('DNS Name of the ACR')
+param acrDnsName string
+
+@description('URL of the pull secret')
+param pullSecretUrl string
+
+@description('URL of the bearer secret')
+param bearerSecretUrl string
+
+resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2022-03-01' existing = {
+  name: environmentName
+}
+
+resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {
+  name: imageSyncManagedIdentity
+}
+
+resource symbolicname 'Microsoft.App/jobs@2024-03-01' = {
+  name: jobName
+  location: location
+
+  identity: {
+    type: 'UserAssigned'
+    userAssignedIdentities: {
+      '${uami.id}': {}
+    }
+  }
+
+  properties: {
+    environmentId: containerAppEnvironment.id
+    configuration: {
+      eventTriggerConfig: {}
+      triggerType: 'Manual'
+      replicaTimeout: 60 * 60
+      registries: [
+        {
+          identity: uami.id
+          server: acrDnsName
+        }
+      ]
+      secrets: [
+        {
+          name: 'pull-secrets'
+          keyVaultUrl: pullSecretUrl
+          identity: uami.id
+        }
+        {
+          name: 'bearer-secret'
+          keyVaultUrl: bearerSecretUrl
+          identity: uami.id
+        }
+      ]
+    }
+    template: {
+      containers: [
+        {
+          name: jobName
+          image: containerImage
+          volumeMounts: [
+            { volumeName: 'pull-secrets-updated', mountPath: '/auth' }
+          ]
+          env: [
+            { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId }
+            { name: 'DOCKER_CONFIG', value: '/auth' }
+          ]
+        }
+      ]
+      initContainers: [
+        {
+          name: 'decodesecrets'
+          image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0'
+          command: [
+            '/bin/sh'
+          ]
+          args: [
+            '-c'
+            'cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cat /tmp/bearer-secret/bearer-secret | base64 -d > /etc/containers/quayio-auth.json'
+          ]
+          volumeMounts: [
+            { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
+            { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' }
+            { volumeName: 'bearer-secret', mountPath: '/tmp/bearer-secret' }
+          ]
+        }
+      ]
+      volumes: [
+        {
+          name: 'pull-secrets-updated'
+          storageType: 'EmptyDir'
+        }
+        {
+          name: 'pull-secrets'
+          storageType: 'Secret'
+          secrets: [
+            { secretRef: 'pull-secrets' }
+          ]
+        }
+        {
+          name: 'bearer-secret'
+          storageType: 'Secret'
+          secrets: [
+            { secretRef: 'bearer-secret' }
+          ]
+        }
+      ]
+    }
+  }
+}
diff --git a/image-sync/deployment/componentSync/mvp-component-sync.bicepparam b/image-sync/deployment/componentSync/mvp-component-sync.bicepparam
@@ -0,0 +1,15 @@
+using 'component-sync.bicep'
+
+param environmentName = 'image-sync-env-sxo4oqbcjiekg'
+
+param jobName = 'component-sync'
+
+param containerImage = 'arohcpdev.azurecr.io/image-sync/component-sync:latest'
+
+param imageSyncManagedIdentity = 'image-sync-sxo4oqbcjiekg'
+
+param acrDnsName = 'arohcpdev.azurecr.io'
+
+param pullSecretUrl = 'https://aro-hcp-dev-global-kv.vault.azure.net/secrets/component-sync-pull-secret'
+
+param bearerSecretUrl = 'https://aro-hcp-dev-global-kv.vault.azure.net/secrets/bearer-secret'
diff --git a/image-sync/oc-mirror/Dockerfile b/image-sync/oc-mirror/Dockerfile
@@ -31,3 +31,5 @@ COPY --chown=0:0 --chmod=755 --from=downloader \
     /usr/local/bin/oc \
     /usr/local/bin/kubectl \
     /usr/local/bin/
+
+ENTRYPOINT ["oc-mirror", "--continue-on-error", "--config", "/etc/oc-mirror/imageset-config.yml"] 
diff --git a/image-sync/oc-mirror/Makefile b/image-sync/oc-mirror/Makefile
@@ -9,6 +9,7 @@ OC_MIRROR_IMAGE_TAGGED ?= $(OC_MIRROR_IMAGE):$(COMMIT)
 build-push: image push
 
 image:
+	cp ../configuration/mvp-oc-mirror.yml config.yml
 	docker build --platform="linux/amd64" -f "./Dockerfile" -t ${OC_MIRROR_IMAGE_TAGGED} .
 
 push: image

diff --git a/image-sync/oc-mirror/config.yml b/image-sync/oc-mirror/config.yml
@@ -0,0 +1,30 @@
+kind: ImageSetConfiguration
+apiVersion: mirror.openshift.io/v1alpha2
+storageConfig:
+  registry:
+    imageURL: arohcpdev.azurecr.io/mirror/oc-mirror-metadata
+    skipTLS: false
+mirror:
+  platform:
+    architectures:
+      - multi
+      - amd64
+    channels:
+      - name: stable-4.16
+        minVersion: 4.16.0
+        maxVersion: 4.16.3
+        type: ocp
+      - name: stable-4.17
+        minVersion: 4.17.0
+        maxVersion: 4.17.0
+        type: ocp
+    graph: true
+  additionalImages:
+  - name: registry.redhat.io/redhat/redhat-operator-index:v4.16
+  - name: registry.redhat.io/redhat/certified-operator-index:v4.16
+  - name: registry.redhat.io/redhat/community-operator-index:v4.16
+  - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.16
+  - name: registry.redhat.io/redhat/redhat-operator-index:v4.17
+  - name: registry.redhat.io/redhat/certified-operator-index:v4.17
+  - name: registry.redhat.io/redhat/community-operator-index:v4.17
+  - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17
diff --git a/tooling/image-sync/Dockerfile b/tooling/image-sync/Dockerfile
@@ -8,5 +8,7 @@ RUN CGO_ENABLED=1 go build -tags=containers_image_openpgp,requirefips .
 FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:ef0dc582fc2a8dd34fbb41341a3a9a1aaa70d4542ff04ce4e33a641e52e4807e
 WORKDIR /
 
+ADD config.yml /app/config.yml
 COPY --from=builder /app/image-sync .
-ENTRYPOINT ["/image-sync"]
+
+CMD ["/image-sync", "-c", "/app/config.yml"]
diff --git a/tooling/image-sync/Makefile b/tooling/image-sync/Makefile
@@ -17,6 +17,7 @@ clean:
 build-push: image push
 
 image:
+	cp ../../image-sync/configuration/mvp-image-sync.yml config.yml
 	docker build --platform="linux/amd64" -f "./Dockerfile" -t ${ARO_HCP_IMAGE_SYNC_IMAGE}:${COMMIT} .
 
 push:

diff --git a/tooling/image-sync/internal/repository.go b/tooling/image-sync/internal/repository.go
@@ -128,13 +128,13 @@ func (q *QuayRegistry) GetTags(ctx context.Context, image string) ([]string, err
 	return tags, nil
 }
 
-type getAccessToken func(context.Context, *azidentity.DefaultAzureCredential) (string, error)
+type getAccessToken func(context.Context, *azidentity.ManagedIdentityCredential) (string, error)
 type getACRUrl func(string) string
 
 // AzureContainerRegistry implements ACR Repository access
 type AzureContainerRegistry struct {
 	acrName      string
-	credential   *azidentity.DefaultAzureCredential
+	credential   *azidentity.ManagedIdentityCredential
 	acrClient    *azcontainerregistry.Client
 	httpClient   *http.Client
 	numberOfTags int
@@ -146,7 +146,9 @@ type AzureContainerRegistry struct {
 
 // NewAzureContainerRegistry creates a new AzureContainerRegistry access client
 func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry {
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	cred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{
+		ID: azidentity.ClientID(cfg.ManagedIdentityClientID),
+	})
 	if err != nil {
 		Log().Fatalf("failed to obtain a credential: %v", err)
 	}
@@ -164,7 +166,7 @@ func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry {
 		numberOfTags: cfg.NumberOfTags,
 		tenantId:     cfg.TenantId,
 
-		getAccessTokenImpl: func(ctx context.Context, dac *azidentity.DefaultAzureCredential) (string, error) {
+		getAccessTokenImpl: func(ctx context.Context, dac *azidentity.ManagedIdentityCredential) (string, error) {
 			accessToken, err := dac.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{"https://management.core.windows.net//.default"}})
 			if err != nil {
 				return "", err

diff --git a/tooling/image-sync/internal/repository_test.go b/tooling/image-sync/internal/repository_test.go
@@ -121,9 +121,9 @@ func TestQuayGetTags(t *testing.T) {
 func TestGetPullSecret(t *testing.T) {
 	acr := AzureContainerRegistry{
 		tenantId:   "test",
-		credential: &azidentity.DefaultAzureCredential{},
+		credential: &azidentity.ManagedIdentityCredential{},
 
-		getAccessTokenImpl: func(ctx context.Context, dac *azidentity.DefaultAzureCredential) (string, error) {
+		getAccessTokenImpl: func(ctx context.Context, dac *azidentity.ManagedIdentityCredential) (string, error) {
 			return "fooBar", nil
 		},
 		getACRUrlImpl: func(acrName string) string {
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,6 +17,7 @@ clean: @@
     build-push: image push
     image:
+    	cp ../../image-sync/configuration/mvp-image-sync.yml config.yml
     	docker build --platform="linux/amd64" -f "./Dockerfile" -t ${ARO_HCP_IMAGE_SYNC_IMAGE}:${COMMIT} .
     push:
@@ Expand Down @@