From 524742f619785073883d8eef7e91d1c0539ef5b0 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Wed, 11 Dec 2024 16:10:22 +0100 Subject: [PATCH 1/3] Make it possible to sync from ACR to ACR --- tooling/image-sync/README.md | 9 +- tooling/image-sync/example.yml | 11 ++- tooling/image-sync/internal/repository.go | 115 +++++++++++++++++++++- tooling/image-sync/internal/sync.go | 100 +++++++++++++++---- tooling/image-sync/main.go | 17 +++- 5 files changed, 219 insertions(+), 33 deletions(-) diff --git a/tooling/image-sync/README.md b/tooling/image-sync/README.md index e77193039..7bb9fa21b 100644 --- a/tooling/image-sync/README.md +++ b/tooling/image-sync/README.md @@ -10,18 +10,21 @@ The main configuration looks like this: repositories: - registry.k8s.io/external-dns/external-dns numberOfTags: 3 -quaySecretfile: /var/run/quay-secret.json -acrRegistry: someregistry.azurecr.io +acrTargetRegistry: someregistry.azurecr.io tenantId: 1ab61791-4b66-4ea4-85ff-aa2c0bf37e57 +secrets: + - registry: registry.k8s.io + secretFile: /secret.txt ``` Explanation: - `repositories` - list of repositories to sync. Do not specify tags, since this utility will sync only the latest tags. - `numberOfTags` - number of tags to sync. The utility will sync the latest `numberOfTags` tags. - `quaySecretfile` - path to the secret file for the Quay registry. -- `acrRegistry` - the target registry. +- `acrTargetRegistry` - the target registry. - `tenantId` - the tenant ID used for authentication with Azure. - `RequestTimeout` - the timeout for the HTTP requests. Default is 10 seconds. +- `secrets` - Array of secrets used for API authentitcation ### quaySecretfile diff --git a/tooling/image-sync/example.yml b/tooling/image-sync/example.yml index 5d301b313..43a19024d 100644 --- a/tooling/image-sync/example.yml +++ b/tooling/image-sync/example.yml @@ -1,8 +1,9 @@ repositories: - registry.k8s.io/external-dns/external-dns - - quay.io/acm-d/rhtap-hypershift-operator - - quay.io/app-sre/uhc-clusters-service -numberOfTags: 40 -quaySecretfile: /home/jboll/workspace/opensource/quay-secret.json -acrRegistry: testing.azurecr.io + - testingone.azurecr.io/azure-cli +numberOfTags: 2 +acrTargetRegistry: testingtrgt.azurecr.io tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408 +secrets: + - registry: testingone.azurecr.io + azureSecretfile: /home/jboll/workspace/opensource/other-secret.json diff --git a/tooling/image-sync/internal/repository.go b/tooling/image-sync/internal/repository.go index 141774d78..a6ca80402 100644 --- a/tooling/image-sync/internal/repository.go +++ b/tooling/image-sync/internal/repository.go @@ -163,13 +163,13 @@ func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry { } } - client, err := azcontainerregistry.NewClient(fmt.Sprintf("https://%s", cfg.AcrRegistry), cred, nil) + client, err := azcontainerregistry.NewClient(fmt.Sprintf("https://%s", cfg.AcrTargetRegistry), cred, nil) if err != nil { Log().Fatalf("failed to create client: %v", err) } return &AzureContainerRegistry{ - acrName: cfg.AcrRegistry, + acrName: cfg.AcrTargetRegistry, acrClient: client, credential: cred, httpClient: &http.Client{Timeout: time.Duration(cfg.RequestTimeout) * time.Second}, @@ -295,18 +295,124 @@ func (a *AzureContainerRegistry) GetTags(ctx context.Context, repository string) return tags, nil } +type ACRWithTokenAuth struct { + httpclient *http.Client + acrName string + numberOftags int + bearerToken string +} + +type AccessSecret struct { + AccessToken string `json:"access_token"` +} + +type rawACRTagResponse struct { + Tags []rawACRTags +} + +type rawACRTags struct { + Name string +} + +func getACRBearerToken(ctx context.Context, secret AzureSecretFile, acrName string) (string, error) { + scope := "repository:*:*" + path := fmt.Sprintf("https://%s/oauth2/token?service=%s&scope=%s", acrName, acrName, scope) + + Log().Debugw("Creating request", "path", path) + req, err := http.NewRequestWithContext(ctx, "GET", path, nil) + req.Header.Add("Authorization", fmt.Sprintf("Basic %s", secret.BasicAuthEncoded())) + if err != nil { + return "", fmt.Errorf("failed to create request: %v", err) + } + + // todo replace with timeout enabled client + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to send request: %v", err) + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response: %v", err) + } + + var accessSecret AccessSecret + + err = json.Unmarshal(body, &accessSecret) + if err != nil { + return "", fmt.Errorf("failed to unmarshal response: %v", err) + } + + return accessSecret.AccessToken, nil +} + +func NewACRWithTokenAuth(cfg *SyncConfig, acrName string, bearerToken string) *ACRWithTokenAuth { + return &ACRWithTokenAuth{ + httpclient: &http.Client{Timeout: time.Duration(cfg.RequestTimeout) * time.Second}, + acrName: acrName, + bearerToken: bearerToken, + numberOftags: cfg.NumberOfTags, + } +} + +func (n *ACRWithTokenAuth) GetTags(ctx context.Context, image string) ([]string, error) { + Log().Debugw("Getting tags for image", "image", image) + + path := fmt.Sprintf("https://%s/acr/v1/%s/_tags?orderby=%s&n=%d", n.acrName, image, azcontainerregistry.ArtifactTagOrderByLastUpdatedOnDescending, n.numberOftags) + req, err := http.NewRequestWithContext(ctx, "GET", path, nil) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", n.bearerToken)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %v", err) + } + + Log().Debugw("Sending request", "path", path) + resp, err := n.httpclient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %v", err) + } + Log().Debugw("Got response", "statuscode", resp.StatusCode) + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %v", err) + } + + var acrResponse rawACRTagResponse + err = json.Unmarshal(body, &acrResponse) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + tagList := make([]string, 0) + + for _, tag := range acrResponse.Tags { + tagList = append(tagList, tag.Name) + } + + return tagList, nil +} + // OCIRegistry implements OCI Repository access type OCIRegistry struct { httpclient *http.Client baseURL string numberOftags int + bearerToken string } // NewOCIRegistry creates a new OCIRegistry access client -func NewOCIRegistry(cfg *SyncConfig, baseURL string) *OCIRegistry { +func NewOCIRegistry(cfg *SyncConfig, baseURL, bearerToken string) *OCIRegistry { o := &OCIRegistry{ httpclient: &http.Client{Timeout: time.Duration(cfg.RequestTimeout) * time.Second}, numberOftags: cfg.NumberOfTags, + bearerToken: bearerToken, } if !strings.HasPrefix(o.baseURL, "https://") { o.baseURL = fmt.Sprintf("https://%s", baseURL) @@ -361,6 +467,9 @@ func (o *OCIRegistry) GetTags(ctx context.Context, image string) ([]string, erro path := fmt.Sprintf("%s/v2/%s/tags/list", o.baseURL, image) req, err := http.NewRequestWithContext(ctx, "GET", path, nil) + if o.bearerToken != "" { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", o.bearerToken)) + } if err != nil { return nil, fmt.Errorf("failed to create request: %v", err) } diff --git a/tooling/image-sync/internal/sync.go b/tooling/image-sync/internal/sync.go index 3e93e73a4..88a7abb6e 100644 --- a/tooling/image-sync/internal/sync.go +++ b/tooling/image-sync/internal/sync.go @@ -2,6 +2,7 @@ package internal import ( "context" + "encoding/base64" "encoding/json" "fmt" "os" @@ -22,19 +23,34 @@ func Log() *zap.SugaredLogger { type SyncConfig struct { Repositories []string NumberOfTags int - QuaySecretFile string - AcrRegistry string + Secrets []Secrets + AcrTargetRegistry string TenantId string RequestTimeout int AddLatest bool ManagedIdentityClientID string } +type Secrets struct { + Registry string + SecretFile string + AzureSecretfile string +} -// QuaySecret is the secret for quay.io -type QuaySecret struct { +// BearerSecret is the secret for the source OCI registry +type BearerSecret struct { BearerToken string } +// AzureSecret is the token configured in the ACR +type AzureSecretFile struct { + Username string + Password string +} + +func (a AzureSecretFile) BasicAuthEncoded() string { + return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.Username, a.Password))) +} + // Copy copies an image from one registry to another func Copy(ctx context.Context, dstreference, srcreference string, dstauth, srcauth *types.DockerAuthConfig) error { policyctx, err := signature.NewPolicyContext(&signature.Policy{ @@ -68,12 +84,26 @@ func Copy(ctx context.Context, dstreference, srcreference string, dstauth, srcau return err } -func readQuaySecret(filename string) (*QuaySecret, error) { +func readBearerSecret(filename string) (*BearerSecret, error) { + secretBytes, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + var secret BearerSecret + err = json.Unmarshal(secretBytes, &secret) + if err != nil { + return nil, err + } + + return &secret, nil +} + +func readAzureSecret(filename string) (*AzureSecretFile, error) { secretBytes, err := os.ReadFile(filename) if err != nil { return nil, err } - var secret QuaySecret + var secret AzureSecretFile err = json.Unmarshal(secretBytes, &secret) if err != nil { return nil, err @@ -103,19 +133,48 @@ func DoSync(cfg *SyncConfig) error { Log().Infow("Syncing images", "images", cfg.Repositories, "numberoftags", cfg.NumberOfTags) ctx := context.Background() - quaySecret, err := readQuaySecret(cfg.QuaySecretFile) - if err != nil { - return fmt.Errorf("error reading secret file: %w %s", err, cfg.QuaySecretFile) + srcRegistries := make(map[string]Registry) + var err error + + for _, secret := range cfg.Secrets { + if secret.Registry == "quay.io" { + quaySecret, err := readBearerSecret(secret.SecretFile) + if err != nil { + return fmt.Errorf("error reading secret file: %w %s", err, secret.SecretFile) + } + qr := NewQuayRegistry(cfg, quaySecret.BearerToken) + srcRegistries[secret.Registry] = qr + } else { + if strings.HasSuffix(secret.Registry, "azurecr.io") || + strings.HasSuffix(secret.Registry, "azurecr.cn") || + strings.HasSuffix(secret.Registry, "azurecr.us") { + azureSecret, err := readAzureSecret(secret.AzureSecretfile) + if err != nil { + return fmt.Errorf("error reading azure secret file: %w %s", err, secret.AzureSecretfile) + } + bearerSecret, err := getACRBearerToken(ctx, *azureSecret, secret.Registry) + if err != nil { + return fmt.Errorf("error getting ACR bearer token: %w", err) + } + srcRegistries[secret.Registry] = NewACRWithTokenAuth(cfg, secret.Registry, bearerSecret) + } else { + s, err := readBearerSecret(secret.SecretFile) + bearerSecret := s.BearerToken + if err != nil { + return fmt.Errorf("error reading secret file: %w %s", err, secret.SecretFile) + } + srcRegistries[secret.Registry] = NewOCIRegistry(cfg, secret.Registry, bearerSecret) + } + } } - qr := NewQuayRegistry(cfg, quaySecret.BearerToken) - acr := NewAzureContainerRegistry(cfg) - acrPullSecret, err := acr.GetPullSecret(ctx) + targetACR := NewAzureContainerRegistry(cfg) + acrPullSecret, err := targetACR.GetPullSecret(ctx) if err != nil { return fmt.Errorf("error getting pull secret: %w", err) } - acrAuth := types.DockerAuthConfig{Username: "00000000-0000-0000-0000-000000000000", Password: acrPullSecret.RefreshToken} + targetACRAuth := types.DockerAuthConfig{Username: "00000000-0000-0000-0000-000000000000", Password: acrPullSecret.RefreshToken} for _, repoName := range cfg.Repositories { var srcTags, acrTags []string @@ -125,14 +184,15 @@ func DoSync(cfg *SyncConfig) error { Log().Infow("Syncing repository", "repository", repoName, "baseurl", baseURL) - if baseURL == "quay.io" { - srcTags, err = qr.GetTags(ctx, repoName) + if client, ok := srcRegistries[baseURL]; ok { + srcTags, err = client.GetTags(ctx, repoName) if err != nil { return fmt.Errorf("error getting quay tags: %w", err) } Log().Debugw("Got tags from quay", "tags", srcTags) } else { - oci := NewOCIRegistry(cfg, baseURL) + // No secret defined, create a default client without auth + oci := NewOCIRegistry(cfg, baseURL, "") srcTags, err = oci.GetTags(ctx, repoName) if err != nil { return fmt.Errorf("error getting oci tags: %w", err) @@ -140,13 +200,13 @@ func DoSync(cfg *SyncConfig) error { Log().Debugw(fmt.Sprintf("Got tags from %s", baseURL), "repo", repoName, "tags", srcTags) } - exists, err := acr.RepositoryExists(ctx, repoName) + exists, err := targetACR.RepositoryExists(ctx, repoName) if err != nil { return fmt.Errorf("error getting ACR repository information: %w", err) } if exists { - acrTags, err = acr.GetTags(ctx, repoName) + acrTags, err = targetACR.GetTags(ctx, repoName) if err != nil { return fmt.Errorf("error getting ACR tags: %w", err) } @@ -161,10 +221,10 @@ func DoSync(cfg *SyncConfig) error { for _, tagToSync := range tagsToSync { source := fmt.Sprintf("%s/%s:%s", baseURL, repoName, tagToSync) - target := fmt.Sprintf("%s/%s:%s", cfg.AcrRegistry, repoName, tagToSync) + target := fmt.Sprintf("%s/%s:%s", cfg.AcrTargetRegistry, repoName, tagToSync) Log().Infow("Copying images", "images", tagToSync, "from", source, "to", target) - err = Copy(ctx, target, source, &acrAuth, nil) + err = Copy(ctx, target, source, &targetACRAuth, nil) if err != nil { return fmt.Errorf("error copying image: %w", err) } diff --git a/tooling/image-sync/main.go b/tooling/image-sync/main.go index e430ccdd1..a632c6bce 100644 --- a/tooling/image-sync/main.go +++ b/tooling/image-sync/main.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" defaultlog "log" "os" "time" @@ -64,8 +65,7 @@ func newSyncConfig() *internal.SyncConfig { "RequestTimeout": "REQUEST_TIMEOUT", "AddLatest": "ADD_LATEST", "Repositories": "REPOSITORIES", - "QuaySecretFile": "QUAY_SECRET_FILE", - "AcrRegistry": "ACR_REGISTRY", + "AcrTargetRegistry": "ACR_TARGET_REGISTRY", "TenantId": "TENANT_ID", "ManagedIdentityClientID": "MANAGED_IDENTITY_CLIENT_ID", } @@ -78,6 +78,19 @@ func newSyncConfig() *internal.SyncConfig { if err := v.Unmarshal(&sc); err != nil { Log().Fatalw("Error while unmarshalling configuration %s", err.Error()) } + + if secretEnv := os.Getenv("SECRETS"); secretEnv != "" { + type listOfSecrets struct { + Secrets []internal.Secrets + } + var s listOfSecrets + err := json.Unmarshal([]byte(secretEnv), &s) + if err != nil { + Log().Fatal("Error unmarshalling configuration") + } + sc.Secrets = append(sc.Secrets, s.Secrets...) + } + Log().Debugw("Using configuration", "config", sc) return sc } From 9ec604adbfe8d3ff97fb6425242692ca44392412 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Wed, 11 Dec 2024 16:24:57 +0100 Subject: [PATCH 2/3] Update env variables --- dev-infrastructure/templates/image-sync.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep index cac74219b..cb67b2d47 100644 --- a/dev-infrastructure/templates/image-sync.bicep +++ b/dev-infrastructure/templates/image-sync.bicep @@ -173,11 +173,11 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = { env: [ { name: 'NUMBER_OF_TAGS', value: '${numberOfTags}' } { name: 'REPOSITORIES', value: repositoriesToSync } - { name: 'QUAY_SECRET_FILE', value: '/auth/${pullSecretFile}' } - { name: 'ACR_REGISTRY', value: '${svcAcrName}${environment().suffixes.acrLoginServer}' } + { name: 'ACR_TARGET_REGISTRY', value: '${svcAcrName}${environment().suffixes.acrLoginServer}' } { name: 'TENANT_ID', value: tenant().tenantId } { name: 'DOCKER_CONFIG', value: '/auth' } { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId } + { name: 'SECRETS', value:'{"secrets":[{"registry": "quay.io", "azureSecretfile": "/auth/${pullSecretFile}"}]}'} ] } ] From f22e867f354c59033ddd1caf3fa233c7c148fd55 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Boll Date: Wed, 11 Dec 2024 16:39:02 +0100 Subject: [PATCH 3/3] Fix format --- dev-infrastructure/templates/image-sync.bicep | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep index cb67b2d47..966bf28a0 100644 --- a/dev-infrastructure/templates/image-sync.bicep +++ b/dev-infrastructure/templates/image-sync.bicep @@ -177,7 +177,10 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = { { name: 'TENANT_ID', value: tenant().tenantId } { name: 'DOCKER_CONFIG', value: '/auth' } { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId } - { name: 'SECRETS', value:'{"secrets":[{"registry": "quay.io", "azureSecretfile": "/auth/${pullSecretFile}"}]}'} + { + name: 'SECRETS' + value: '{"secrets":[{"registry": "quay.io", "azureSecretfile": "/auth/${pullSecretFile}"}]}' + } ] } ]