From 62a464c622ae57237e172428a155fff2deed2643 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Tue, 15 Oct 2024 10:06:48 +0200 Subject: [PATCH 1/7] Change Container builds Add the configuration to this repo and reference it in the Dockerfile, thus add it to the image --- image-sync/configuration/mvp-image-sync.yml | 9 +++++++ image-sync/configuration/mvp-oc-mirror.yml | 30 +++++++++++++++++++++ image-sync/oc-mirror/Dockerfile | 2 ++ image-sync/oc-mirror/Makefile | 1 + image-sync/oc-mirror/config.yml | 30 +++++++++++++++++++++ tooling/image-sync/Dockerfile | 3 ++- tooling/image-sync/Makefile | 1 + tooling/image-sync/config.yml | 9 +++++++ 8 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 image-sync/configuration/mvp-image-sync.yml create mode 100644 image-sync/configuration/mvp-oc-mirror.yml create mode 100644 image-sync/oc-mirror/config.yml create mode 100644 tooling/image-sync/config.yml diff --git a/image-sync/configuration/mvp-image-sync.yml b/image-sync/configuration/mvp-image-sync.yml new file mode 100644 index 000000000..3dd45c233 --- /dev/null +++ b/image-sync/configuration/mvp-image-sync.yml @@ -0,0 +1,9 @@ +repositories: + - registry.k8s.io/external-dns/external-dns + - quay.io/acm-d/rhtap-hypershift-operator + - quay.io/pstefans/controlplaneoperator + - quay.io/app-sre/uhc-clusters-service +numberOfTags: 10 +quaySecretfile: /etc/containers/quayio-auth.json +acrRegistry: arohcpdev.azurecr.io +tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408 diff --git a/image-sync/configuration/mvp-oc-mirror.yml b/image-sync/configuration/mvp-oc-mirror.yml new file mode 100644 index 000000000..483d847ce --- /dev/null +++ b/image-sync/configuration/mvp-oc-mirror.yml @@ -0,0 +1,30 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v1alpha2 +storageConfig: + registry: + imageURL: arohcpdev.azurecr.io/mirror/oc-mirror-metadata + skipTLS: false +mirror: + platform: + architectures: + - multi + - amd64 + channels: + - name: stable-4.16 + minVersion: 4.16.0 + maxVersion: 4.16.3 + type: ocp + - name: stable-4.17 + minVersion: 4.17.0 + maxVersion: 4.17.0 + type: ocp + graph: true + additionalImages: + - name: registry.redhat.io/redhat/redhat-operator-index:v4.16 + - name: registry.redhat.io/redhat/certified-operator-index:v4.16 + - name: registry.redhat.io/redhat/community-operator-index:v4.16 + - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.16 + - name: registry.redhat.io/redhat/redhat-operator-index:v4.17 + - name: registry.redhat.io/redhat/certified-operator-index:v4.17 + - name: registry.redhat.io/redhat/community-operator-index:v4.17 + - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 \ No newline at end of file diff --git a/image-sync/oc-mirror/Dockerfile b/image-sync/oc-mirror/Dockerfile index 4859eef40..f307dda62 100644 --- a/image-sync/oc-mirror/Dockerfile +++ b/image-sync/oc-mirror/Dockerfile @@ -31,3 +31,5 @@ COPY --chown=0:0 --chmod=755 --from=downloader \ /usr/local/bin/oc \ /usr/local/bin/kubectl \ /usr/local/bin/ + +ENTRYPOINT ["oc-mirror", "--continue-on-error", "--config", "/etc/oc-mirror/imageset-config.yml"] diff --git a/image-sync/oc-mirror/Makefile b/image-sync/oc-mirror/Makefile index 08e80618c..e62cfc6db 100644 --- a/image-sync/oc-mirror/Makefile +++ b/image-sync/oc-mirror/Makefile @@ -9,6 +9,7 @@ OC_MIRROR_IMAGE_TAGGED ?= $(OC_MIRROR_IMAGE):$(COMMIT) build-push: image push image: + cp ../configuration/mvp-oc-mirror.yml config.yml docker build --platform="linux/amd64" -f "./Dockerfile" -t ${OC_MIRROR_IMAGE_TAGGED} . push: image diff --git a/image-sync/oc-mirror/config.yml b/image-sync/oc-mirror/config.yml new file mode 100644 index 000000000..483d847ce --- /dev/null +++ b/image-sync/oc-mirror/config.yml @@ -0,0 +1,30 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v1alpha2 +storageConfig: + registry: + imageURL: arohcpdev.azurecr.io/mirror/oc-mirror-metadata + skipTLS: false +mirror: + platform: + architectures: + - multi + - amd64 + channels: + - name: stable-4.16 + minVersion: 4.16.0 + maxVersion: 4.16.3 + type: ocp + - name: stable-4.17 + minVersion: 4.17.0 + maxVersion: 4.17.0 + type: ocp + graph: true + additionalImages: + - name: registry.redhat.io/redhat/redhat-operator-index:v4.16 + - name: registry.redhat.io/redhat/certified-operator-index:v4.16 + - name: registry.redhat.io/redhat/community-operator-index:v4.16 + - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.16 + - name: registry.redhat.io/redhat/redhat-operator-index:v4.17 + - name: registry.redhat.io/redhat/certified-operator-index:v4.17 + - name: registry.redhat.io/redhat/community-operator-index:v4.17 + - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 \ No newline at end of file diff --git a/tooling/image-sync/Dockerfile b/tooling/image-sync/Dockerfile index 7ff6cc491..94b60e058 100644 --- a/tooling/image-sync/Dockerfile +++ b/tooling/image-sync/Dockerfile @@ -8,5 +8,6 @@ RUN CGO_ENABLED=1 go build -tags=containers_image_openpgp,requirefips . FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:ef0dc582fc2a8dd34fbb41341a3a9a1aaa70d4542ff04ce4e33a641e52e4807e WORKDIR / +ADD config.yml /app/config.yml COPY --from=builder /app/image-sync . -ENTRYPOINT ["/image-sync"] \ No newline at end of file +CMD ["/image-sync", "-c", "/app/config.yml"] \ No newline at end of file diff --git a/tooling/image-sync/Makefile b/tooling/image-sync/Makefile index 348db37ed..ee5c42683 100644 --- a/tooling/image-sync/Makefile +++ b/tooling/image-sync/Makefile @@ -17,6 +17,7 @@ clean: build-push: image push image: + cp ../../image-sync/configuration/mvp-image-sync.yml config.yml docker build --platform="linux/amd64" -f "./Dockerfile" -t ${ARO_HCP_IMAGE_SYNC_IMAGE}:${COMMIT} . push: diff --git a/tooling/image-sync/config.yml b/tooling/image-sync/config.yml new file mode 100644 index 000000000..3dd45c233 --- /dev/null +++ b/tooling/image-sync/config.yml @@ -0,0 +1,9 @@ +repositories: + - registry.k8s.io/external-dns/external-dns + - quay.io/acm-d/rhtap-hypershift-operator + - quay.io/pstefans/controlplaneoperator + - quay.io/app-sre/uhc-clusters-service +numberOfTags: 10 +quaySecretfile: /etc/containers/quayio-auth.json +acrRegistry: arohcpdev.azurecr.io +tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408 From aea419fed2c5f78c1333e8db11497c676fd793d7 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Wed, 23 Oct 2024 14:27:18 +0200 Subject: [PATCH 2/7] Make component sync work on container apps --- dev-infrastructure/configurations/dev.mk | 2 +- image-sync/configuration/mvp-image-sync.yml | 2 +- image-sync/configuration/mvp-oc-mirror.yml | 2 +- image-sync/deployment/Makefile | 6 +- .../componentSync/mvp-componentSyncJob.yml | 56 +++++++++++++++++++ image-sync/oc-mirror/config.yml | 2 +- tooling/image-sync/Dockerfile | 1 + tooling/image-sync/config.yml | 9 --- tooling/image-sync/internal/repository.go | 10 ++-- .../image-sync/internal/repository_test.go | 4 +- tooling/image-sync/internal/sync.go | 19 ++++--- 11 files changed, 86 insertions(+), 27 deletions(-) create mode 100644 image-sync/deployment/componentSync/mvp-componentSyncJob.yml delete mode 100644 tooling/image-sync/config.yml diff --git a/dev-infrastructure/configurations/dev.mk b/dev-infrastructure/configurations/dev.mk index b6ffea374..ce757e192 100644 --- a/dev-infrastructure/configurations/dev.mk +++ b/dev-infrastructure/configurations/dev.mk @@ -4,6 +4,6 @@ REGIONAL_RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION) SVC_KV_RESOURCEGROUP ?= global GLOBAL_RESOURCEGROUP ?= global IMAGE_SYNC_RESOURCEGROUP ?= aro-hcp-image-sync-$(USER)-$(REGION) -IMAGE_SYNC_ENVIRONMENT ?= image-sync-env +IMAGE_SYNC_ENVIRONMENT ?= image-sync-env-sxo4oqbcjiekg ARO_HCP_IMAGE_ACR ?= arohcpdev REPOSITORIES_TO_SYNC ?= '{registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/pstefans/controlplaneoperator,quay.io/app-sre/uhc-clusters-service}' diff --git a/image-sync/configuration/mvp-image-sync.yml b/image-sync/configuration/mvp-image-sync.yml index 3dd45c233..4a0b7339c 100644 --- a/image-sync/configuration/mvp-image-sync.yml +++ b/image-sync/configuration/mvp-image-sync.yml @@ -4,6 +4,6 @@ repositories: - quay.io/pstefans/controlplaneoperator - quay.io/app-sre/uhc-clusters-service numberOfTags: 10 -quaySecretfile: /etc/containers/quayio-auth.json +quaySecretfile: /root/.docker/quayio-auth.json acrRegistry: arohcpdev.azurecr.io tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408 diff --git a/image-sync/configuration/mvp-oc-mirror.yml b/image-sync/configuration/mvp-oc-mirror.yml index 483d847ce..577f6cdf7 100644 --- a/image-sync/configuration/mvp-oc-mirror.yml +++ b/image-sync/configuration/mvp-oc-mirror.yml @@ -27,4 +27,4 @@ mirror: - name: registry.redhat.io/redhat/redhat-operator-index:v4.17 - name: registry.redhat.io/redhat/certified-operator-index:v4.17 - name: registry.redhat.io/redhat/community-operator-index:v4.17 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 \ No newline at end of file + - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 diff --git a/image-sync/deployment/Makefile b/image-sync/deployment/Makefile index a58f1b240..0aebc063c 100644 --- a/image-sync/deployment/Makefile +++ b/image-sync/deployment/Makefile @@ -63,4 +63,8 @@ undeploy-oc-mirror: undeploy: undeploy-shared undeploy-component-sync undeploy-oc-mirror -.PHONY: deploy-component-sync deploy-shared deploy-oc-mirror undeploy-shared undeploy-component-sync undeploy-oc-mirror +deploy-ca-component-sync: + az containerapp job create -n component-sync-job -g ${IMAGE_SYNC_RESOURCEGROUP} \ + --yaml ./componentSync/mvp-componentSyncJob.yml + +.PHONY: deploy-caj-component-sync deploy-component-sync deploy-shared deploy-oc-mirror undeploy-shared undeploy-component-sync undeploy-oc-mirror diff --git a/image-sync/deployment/componentSync/mvp-componentSyncJob.yml b/image-sync/deployment/componentSync/mvp-componentSyncJob.yml new file mode 100644 index 000000000..7c63f81e4 --- /dev/null +++ b/image-sync/deployment/componentSync/mvp-componentSyncJob.yml @@ -0,0 +1,56 @@ +identity: + userAssignedIdentities: + "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg": {} + type: UserAssigned +properties: + environmentId: "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/aro-hcp-dev-image-sync/providers/Microsoft.App/managedEnvironments/image-sync-env-sxo4oqbcjiekg" + configuration: + replicaTimeout: 10000 + replicaRetryLimit: 1 + manualTriggerConfig: + replicaCompletionCount: 1 + parallelism: 1 + triggerType: Manual + registries: + - identity: "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg" + server: arohcpdev.azurecr.io + secrets: + - name: pull-secrets + keyVaultUrl: https://aro-hcp-dev-global-kv.vault.azure.net/secrets/component-sync-pull-secret + identity: /subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg + - name: bearer-secret + keyVaultUrl: https://aro-hcp-dev-global-kv.vault.azure.net/secrets/bearer-secret + identity: /subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg + template: + containers: + - image: arohcpdev.azurecr.io/image-sync/component-sync:latest + name: sync-components + volumeMounts: + - volumeName: pull-secrets-updated + mountPath: "/root/.docker" + initContainers: + - image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 + name: decodesecrets + command: + - "/bin/sh" + args: + - "-c" + - "cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cat /tmp/bearer-secret/bearer-secret | base64 -d > /etc/containers/quayio-auth.json" + volumeMounts: + - volumeName: pull-secrets-updated + mountPath: "/etc/containers" + - volumeName: pull-secrets + mountPath: "/tmp/secret-orig" + - volumeName: bearer-secret + mountPath: "/tmp/bearer-secret" + volumes: + - name: pull-secrets-updated + storageType: EmptyDir + - name: pull-secrets + storageType: Secret + secrets: + - secretRef: pull-secrets + - name: bearer-secret + storageType: Secret + secrets: + - secretRef: bearer-secret diff --git a/image-sync/oc-mirror/config.yml b/image-sync/oc-mirror/config.yml index 483d847ce..577f6cdf7 100644 --- a/image-sync/oc-mirror/config.yml +++ b/image-sync/oc-mirror/config.yml @@ -27,4 +27,4 @@ mirror: - name: registry.redhat.io/redhat/redhat-operator-index:v4.17 - name: registry.redhat.io/redhat/certified-operator-index:v4.17 - name: registry.redhat.io/redhat/community-operator-index:v4.17 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 \ No newline at end of file + - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 diff --git a/tooling/image-sync/Dockerfile b/tooling/image-sync/Dockerfile index 94b60e058..2b0a8a1c9 100644 --- a/tooling/image-sync/Dockerfile +++ b/tooling/image-sync/Dockerfile @@ -10,4 +10,5 @@ WORKDIR / ADD config.yml /app/config.yml COPY --from=builder /app/image-sync . + CMD ["/image-sync", "-c", "/app/config.yml"] \ No newline at end of file diff --git a/tooling/image-sync/config.yml b/tooling/image-sync/config.yml deleted file mode 100644 index 3dd45c233..000000000 --- a/tooling/image-sync/config.yml +++ /dev/null @@ -1,9 +0,0 @@ -repositories: - - registry.k8s.io/external-dns/external-dns - - quay.io/acm-d/rhtap-hypershift-operator - - quay.io/pstefans/controlplaneoperator - - quay.io/app-sre/uhc-clusters-service -numberOfTags: 10 -quaySecretfile: /etc/containers/quayio-auth.json -acrRegistry: arohcpdev.azurecr.io -tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408 diff --git a/tooling/image-sync/internal/repository.go b/tooling/image-sync/internal/repository.go index 7ee512f0b..668eb8358 100644 --- a/tooling/image-sync/internal/repository.go +++ b/tooling/image-sync/internal/repository.go @@ -128,13 +128,13 @@ func (q *QuayRegistry) GetTags(ctx context.Context, image string) ([]string, err return tags, nil } -type getAccessToken func(context.Context, *azidentity.DefaultAzureCredential) (string, error) +type getAccessToken func(context.Context, *azidentity.ManagedIdentityCredential) (string, error) type getACRUrl func(string) string // AzureContainerRegistry implements ACR Repository access type AzureContainerRegistry struct { acrName string - credential *azidentity.DefaultAzureCredential + credential *azidentity.ManagedIdentityCredential acrClient *azcontainerregistry.Client httpClient *http.Client numberOfTags int @@ -146,7 +146,9 @@ type AzureContainerRegistry struct { // NewAzureContainerRegistry creates a new AzureContainerRegistry access client func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry { - cred, err := azidentity.NewDefaultAzureCredential(nil) + cred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(cfg.ManagedIdentityClientID), + }) if err != nil { Log().Fatalf("failed to obtain a credential: %v", err) } @@ -164,7 +166,7 @@ func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry { numberOfTags: cfg.NumberOfTags, tenantId: cfg.TenantId, - getAccessTokenImpl: func(ctx context.Context, dac *azidentity.DefaultAzureCredential) (string, error) { + getAccessTokenImpl: func(ctx context.Context, dac *azidentity.ManagedIdentityCredential) (string, error) { accessToken, err := dac.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{"https://management.core.windows.net//.default"}}) if err != nil { return "", err diff --git a/tooling/image-sync/internal/repository_test.go b/tooling/image-sync/internal/repository_test.go index dbda141f5..bd3fd1264 100644 --- a/tooling/image-sync/internal/repository_test.go +++ b/tooling/image-sync/internal/repository_test.go @@ -121,9 +121,9 @@ func TestQuayGetTags(t *testing.T) { func TestGetPullSecret(t *testing.T) { acr := AzureContainerRegistry{ tenantId: "test", - credential: &azidentity.DefaultAzureCredential{}, + credential: &azidentity.ManagedIdentityCredential{}, - getAccessTokenImpl: func(ctx context.Context, dac *azidentity.DefaultAzureCredential) (string, error) { + getAccessTokenImpl: func(ctx context.Context, dac *azidentity.ManagedIdentityCredential) (string, error) { return "fooBar", nil }, getACRUrlImpl: func(acrName string) string { diff --git a/tooling/image-sync/internal/sync.go b/tooling/image-sync/internal/sync.go index af1dfe6c0..8f6e79569 100644 --- a/tooling/image-sync/internal/sync.go +++ b/tooling/image-sync/internal/sync.go @@ -21,13 +21,14 @@ func Log() *zap.SugaredLogger { // SyncConfig is the configuration for the image sync type SyncConfig struct { - Repositories []string - NumberOfTags int - QuaySecretFile string - AcrRegistry string - TenantId string - RequestTimeout int - AddLatest bool + Repositories []string + NumberOfTags int + QuaySecretFile string + AcrRegistry string + TenantId string + RequestTimeout int + AddLatest bool + ManagedIdentityClientID string } // QuaySecret is the secret for quay.io @@ -43,6 +44,10 @@ func NewSyncConfig() *SyncConfig { v.SetDefault("requesttimeout", 10) v.SetDefault("addlatest", false) + if err := v.BindEnv("ManagedIdentityClientId", "MANAGED_IDENTITY_CLIENT_ID"); err != nil { + Log().Fatalw("Error while binding environment variable %s", err.Error()) + } + if err := v.Unmarshal(&sc); err != nil { Log().Fatalw("Error while unmarshalling configuration %s", err.Error()) } From 0ad7c738dcc431369f51fe2b6b87254e9ec78fcc Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Thu, 24 Oct 2024 10:40:55 +0200 Subject: [PATCH 3/7] Refactor to use bicep Using bicep makes it easier to re-use environment and UAMI created out of this deployment scope. --- image-sync/deployment/Makefile | 47 +++---- .../componentSync/component-sync.bicep | 122 ++++++++++++++++++ .../mvp-component-sync.bicepparam | 15 +++ .../componentSync/mvp-componentSyncJob.yml | 56 -------- 4 files changed, 162 insertions(+), 78 deletions(-) create mode 100644 image-sync/deployment/componentSync/component-sync.bicep create mode 100644 image-sync/deployment/componentSync/mvp-component-sync.bicepparam delete mode 100644 image-sync/deployment/componentSync/mvp-componentSyncJob.yml diff --git a/image-sync/deployment/Makefile b/image-sync/deployment/Makefile index 0aebc063c..2a72be4b6 100644 --- a/image-sync/deployment/Makefile +++ b/image-sync/deployment/Makefile @@ -14,24 +14,24 @@ deploy-shared: --set serviceAccountName=image-sync \ --set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID} -deploy-component-sync: - AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n image-sync \ - --query clientId) && \ - TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - helm upgrade --force --install component-sync-cronjob ./componentSyncCronjob \ - --namespace ${NAMESPACE} --create-namespace \ - --set serviceAccountName=image-sync \ - --set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID} \ - --set acrRegistryName=${ARO_HCP_IMAGE_ACR} \ - --set azureTenantId=$${TENANT_ID} \ - --set componentSyncImage="arohcpdev.azurecr.io/image-sync/component-sync" \ - --set componentSyncTag=latest \ - --set credsBearerSecret=bearer-secret \ - --set credsPullSecret=component-pull-secret \ - --set credsKeyVaultName=service-kv-aro-hcp-dev \ - --set repositories=${REPOSITORIES_TO_SYNC} +# deploy-component-sync: +# AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \ +# -g ${RESOURCEGROUP} \ +# -n image-sync \ +# --query clientId) && \ +# TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ +# helm upgrade --force --install component-sync-cronjob ./componentSyncCronjob \ +# --namespace ${NAMESPACE} --create-namespace \ +# --set serviceAccountName=image-sync \ +# --set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID} \ +# --set acrRegistryName=${ARO_HCP_IMAGE_ACR} \ +# --set azureTenantId=$${TENANT_ID} \ +# --set componentSyncImage="arohcpdev.azurecr.io/image-sync/component-sync" \ +# --set componentSyncTag=latest \ +# --set credsBearerSecret=bearer-secret \ +# --set credsPullSecret=component-pull-secret \ +# --set credsKeyVaultName=service-kv-aro-hcp-dev \ +# --set repositories=${REPOSITORIES_TO_SYNC} deploy-oc-mirror: AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \ @@ -63,8 +63,11 @@ undeploy-oc-mirror: undeploy: undeploy-shared undeploy-component-sync undeploy-oc-mirror -deploy-ca-component-sync: - az containerapp job create -n component-sync-job -g ${IMAGE_SYNC_RESOURCEGROUP} \ - --yaml ./componentSync/mvp-componentSyncJob.yml +deploy-component-sync: + az deployment group create --name comp-sync \ + --resource-group $IMAGE_SYNC_RESOURCEGROUP \ + --template-file componentSync/component-sync.bicep \ + --parameters componentSync/mvp-component-sync.bicepparam \ + --parameter containerImage="arohcpdev.azurecr.io/image-sync/component-sync:testing" -.PHONY: deploy-caj-component-sync deploy-component-sync deploy-shared deploy-oc-mirror undeploy-shared undeploy-component-sync undeploy-oc-mirror +.PHONY: deploy-component-sync deploy-shared deploy-oc-mirror undeploy-shared undeploy-component-sync undeploy-oc-mirror diff --git a/image-sync/deployment/componentSync/component-sync.bicep b/image-sync/deployment/componentSync/component-sync.bicep new file mode 100644 index 000000000..78d9fc768 --- /dev/null +++ b/image-sync/deployment/componentSync/component-sync.bicep @@ -0,0 +1,122 @@ +@description('Azure Region Location') +param location string = resourceGroup().location + +@description('Name of the Container App Environment') +param environmentName string + +@description('Name of the Container App Job') +param jobName string + +@description('Container image to use for the job') +param containerImage string + +@description('Name of the user assigned managed identity') +param imageSyncManagedIdentity string + +@description('DNS Name of the ACR') +param acrDnsName string + +@description('URL of the pull secret') +param pullSecretUrl string + +@description('URL of the bearer secret') +param bearerSecretUrl string + +resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2022-03-01' existing = { + name: environmentName +} + +resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { + name: imageSyncManagedIdentity +} + +resource symbolicname 'Microsoft.App/jobs@2024-03-01' = { + name: jobName + location: location + + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${uami.id}': {} + } + } + + properties: { + environmentId: containerAppEnvironment.id + configuration: { + eventTriggerConfig: {} + triggerType: 'Manual' + replicaTimeout: 60 * 60 + registries: [ + { + identity: uami.id + server: acrDnsName + } + ] + secrets: [ + { + name: 'pull-secrets' + keyVaultUrl: pullSecretUrl + identity: uami.id + } + { + name: 'bearer-secret' + keyVaultUrl: bearerSecretUrl + identity: uami.id + } + ] + } + template: { + containers: [ + { + name: jobName + image: containerImage + volumeMounts: [ + { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' } + ] + env: [ + { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId } + ] + } + ] + initContainers: [ + { + name: 'decodesecrets' + image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0' + command: [ + '/bin/sh' + ] + args: [ + '-c' + 'cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cat /tmp/bearer-secret/bearer-secret | base64 -d > /etc/containers/quayio-auth.json' + ] + volumeMounts: [ + { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' } + { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' } + { volumeName: 'bearer-secret', mountPath: '/tmp/bearer-secret' } + ] + } + ] + volumes: [ + { + name: 'pull-secrets-updated' + storageType: 'EmptyDir' + } + { + name: 'pull-secrets' + storageType: 'Secret' + secrets: [ + { secretRef: 'pull-secrets' } + ] + } + { + name: 'bearer-secret' + storageType: 'Secret' + secrets: [ + { secretRef: 'bearer-secret' } + ] + } + ] + } + } +} diff --git a/image-sync/deployment/componentSync/mvp-component-sync.bicepparam b/image-sync/deployment/componentSync/mvp-component-sync.bicepparam new file mode 100644 index 000000000..303767f00 --- /dev/null +++ b/image-sync/deployment/componentSync/mvp-component-sync.bicepparam @@ -0,0 +1,15 @@ +using 'component-sync.bicep' + +param environmentName = 'image-sync-env-sxo4oqbcjiekg' + +param jobName = 'component-sync-aaa' + +param containerImage = 'arohcpdev.azurecr.io/image-sync/component-sync:latest' + +param imageSyncManagedIdentity = 'image-sync-sxo4oqbcjiekg' + +param acrDnsName = 'arohcpdev.azurecr.io' + +param pullSecretUrl = 'https://aro-hcp-dev-global-kv.vault.azure.net/secrets/component-sync-pull-secret' + +param bearerSecretUrl = 'https://aro-hcp-dev-global-kv.vault.azure.net/secrets/bearer-secret' diff --git a/image-sync/deployment/componentSync/mvp-componentSyncJob.yml b/image-sync/deployment/componentSync/mvp-componentSyncJob.yml deleted file mode 100644 index 7c63f81e4..000000000 --- a/image-sync/deployment/componentSync/mvp-componentSyncJob.yml +++ /dev/null @@ -1,56 +0,0 @@ -identity: - userAssignedIdentities: - "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg": {} - type: UserAssigned -properties: - environmentId: "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/aro-hcp-dev-image-sync/providers/Microsoft.App/managedEnvironments/image-sync-env-sxo4oqbcjiekg" - configuration: - replicaTimeout: 10000 - replicaRetryLimit: 1 - manualTriggerConfig: - replicaCompletionCount: 1 - parallelism: 1 - triggerType: Manual - registries: - - identity: "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg" - server: arohcpdev.azurecr.io - secrets: - - name: pull-secrets - keyVaultUrl: https://aro-hcp-dev-global-kv.vault.azure.net/secrets/component-sync-pull-secret - identity: /subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg - - name: bearer-secret - keyVaultUrl: https://aro-hcp-dev-global-kv.vault.azure.net/secrets/bearer-secret - identity: /subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/aro-hcp-dev-image-sync/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-sync-sxo4oqbcjiekg - template: - containers: - - image: arohcpdev.azurecr.io/image-sync/component-sync:latest - name: sync-components - volumeMounts: - - volumeName: pull-secrets-updated - mountPath: "/root/.docker" - initContainers: - - image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 - name: decodesecrets - command: - - "/bin/sh" - args: - - "-c" - - "cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cat /tmp/bearer-secret/bearer-secret | base64 -d > /etc/containers/quayio-auth.json" - volumeMounts: - - volumeName: pull-secrets-updated - mountPath: "/etc/containers" - - volumeName: pull-secrets - mountPath: "/tmp/secret-orig" - - volumeName: bearer-secret - mountPath: "/tmp/bearer-secret" - volumes: - - name: pull-secrets-updated - storageType: EmptyDir - - name: pull-secrets - storageType: Secret - secrets: - - secretRef: pull-secrets - - name: bearer-secret - storageType: Secret - secrets: - - secretRef: bearer-secret From a6b3e9a2fc1c0e74f7e70e10499d4e2c588654b2 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Thu, 24 Oct 2024 14:24:57 +0200 Subject: [PATCH 4/7] Fixes for running the sync --- image-sync/configuration/mvp-image-sync.yml | 2 +- image-sync/deployment/Makefile | 4 ++-- image-sync/deployment/componentSync/component-sync.bicep | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/image-sync/configuration/mvp-image-sync.yml b/image-sync/configuration/mvp-image-sync.yml index 4a0b7339c..bc6d9830c 100644 --- a/image-sync/configuration/mvp-image-sync.yml +++ b/image-sync/configuration/mvp-image-sync.yml @@ -4,6 +4,6 @@ repositories: - quay.io/pstefans/controlplaneoperator - quay.io/app-sre/uhc-clusters-service numberOfTags: 10 -quaySecretfile: /root/.docker/quayio-auth.json +quaySecretfile: /auth/quayio-auth.json acrRegistry: arohcpdev.azurecr.io tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408 diff --git a/image-sync/deployment/Makefile b/image-sync/deployment/Makefile index 2a72be4b6..169185e91 100644 --- a/image-sync/deployment/Makefile +++ b/image-sync/deployment/Makefile @@ -65,9 +65,9 @@ undeploy: undeploy-shared undeploy-component-sync undeploy-oc-mirror deploy-component-sync: az deployment group create --name comp-sync \ - --resource-group $IMAGE_SYNC_RESOURCEGROUP \ + --resource-group ${IMAGE_SYNC_RESOURCEGROUP} \ --template-file componentSync/component-sync.bicep \ --parameters componentSync/mvp-component-sync.bicepparam \ - --parameter containerImage="arohcpdev.azurecr.io/image-sync/component-sync:testing" + --parameters containerImage="arohcpdev.azurecr.io/image-sync/component-sync:latest" .PHONY: deploy-component-sync deploy-shared deploy-oc-mirror undeploy-shared undeploy-component-sync undeploy-oc-mirror diff --git a/image-sync/deployment/componentSync/component-sync.bicep b/image-sync/deployment/componentSync/component-sync.bicep index 78d9fc768..b51a3f026 100644 --- a/image-sync/deployment/componentSync/component-sync.bicep +++ b/image-sync/deployment/componentSync/component-sync.bicep @@ -72,10 +72,11 @@ resource symbolicname 'Microsoft.App/jobs@2024-03-01' = { name: jobName image: containerImage volumeMounts: [ - { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' } + { volumeName: 'pull-secrets-updated', mountPath: '/auth' } ] env: [ { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId } + { name: 'DOCKER_CONFIG', value: '/auth' } ] } ] From 181a5a51237f5d9aaa7efce9bd7ed0c78059c3ed Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Thu, 24 Oct 2024 14:34:06 +0200 Subject: [PATCH 5/7] Add missing new line --- tooling/image-sync/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling/image-sync/Dockerfile b/tooling/image-sync/Dockerfile index 2b0a8a1c9..d188d3134 100644 --- a/tooling/image-sync/Dockerfile +++ b/tooling/image-sync/Dockerfile @@ -11,4 +11,4 @@ WORKDIR / ADD config.yml /app/config.yml COPY --from=builder /app/image-sync . -CMD ["/image-sync", "-c", "/app/config.yml"] \ No newline at end of file +CMD ["/image-sync", "-c", "/app/config.yml"] From ce1d1a2bb3cf0ba7840284c0a59a2cd209d6350d Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Thu, 24 Oct 2024 14:34:17 +0200 Subject: [PATCH 6/7] Fix permissions --- .../configurations/mvp-image-sync.bicepparam | 2 +- dev-infrastructure/templates/image-sync.bicep | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dev-infrastructure/configurations/mvp-image-sync.bicepparam b/dev-infrastructure/configurations/mvp-image-sync.bicepparam index eb12ea165..1dc16bae5 100644 --- a/dev-infrastructure/configurations/mvp-image-sync.bicepparam +++ b/dev-infrastructure/configurations/mvp-image-sync.bicepparam @@ -5,6 +5,6 @@ param acrResourceGroup = 'gobal' param keyVaultName = 'aro-hcp-dev-global-kv' param requiredSecretNames = [ - 'pull-secret' + 'component-sync-pull-secret' 'bearer-secret' ] diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep index e6d20ea18..5c4daec2d 100644 --- a/dev-infrastructure/templates/image-sync.bicep +++ b/dev-infrastructure/templates/image-sync.bicep @@ -53,6 +53,7 @@ resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { module acrContributorRole '../modules/acr-permissions.bicep' = { name: guid(imageSyncManagedIdentity, 'acr', 'readwrite') + scope: resourceGroup(acrResourceGroup) params: { principalId: uami.properties.principalId grantPushAccess: true @@ -60,6 +61,15 @@ module acrContributorRole '../modules/acr-permissions.bicep' = { } } +module acrPullRole '../modules/acr-permissions.bicep' = { + name: guid(imageSyncManagedIdentity, 'acr', 'pull') + scope: resourceGroup(acrResourceGroup) + params: { + principalId: uami.properties.principalId + acrResourceGroupid: acrResourceGroup + } +} + module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = [ for secretName in requiredSecretNames: { name: '${secretName}-access' From e9917e08c7bedc0bf556d822808fb46774edd722 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Thu, 24 Oct 2024 16:00:04 +0200 Subject: [PATCH 7/7] Additional fixes --- dev-infrastructure/configurations/mvp-image-sync.bicepparam | 2 +- .../deployment/componentSync/mvp-component-sync.bicepparam | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-infrastructure/configurations/mvp-image-sync.bicepparam b/dev-infrastructure/configurations/mvp-image-sync.bicepparam index 1dc16bae5..3b58c091a 100644 --- a/dev-infrastructure/configurations/mvp-image-sync.bicepparam +++ b/dev-infrastructure/configurations/mvp-image-sync.bicepparam @@ -1,6 +1,6 @@ using '../templates/image-sync.bicep' -param acrResourceGroup = 'gobal' +param acrResourceGroup = 'global' param keyVaultName = 'aro-hcp-dev-global-kv' diff --git a/image-sync/deployment/componentSync/mvp-component-sync.bicepparam b/image-sync/deployment/componentSync/mvp-component-sync.bicepparam index 303767f00..ab5cf48f4 100644 --- a/image-sync/deployment/componentSync/mvp-component-sync.bicepparam +++ b/image-sync/deployment/componentSync/mvp-component-sync.bicepparam @@ -2,7 +2,7 @@ using 'component-sync.bicep' param environmentName = 'image-sync-env-sxo4oqbcjiekg' -param jobName = 'component-sync-aaa' +param jobName = 'component-sync' param containerImage = 'arohcpdev.azurecr.io/image-sync/component-sync:latest'