Merge pull request #963 from Azure/enable-acr-to-acr

Make it possible to sync from ACR to ACR

dev-infrastructure/templates/image-sync.bicep

@@ -195,11 +195,14 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = if (componentSyncEna
           env: [
             { name: 'NUMBER_OF_TAGS', value: '${numberOfTags}' }
             { name: 'REPOSITORIES', value: repositoriesToSync }
-            { name: 'QUAY_SECRET_FILE', value: '/auth/${pullSecretFile}' }
-            { name: 'ACR_REGISTRY', value: '${svcAcrName}${environment().suffixes.acrLoginServer}' }
+            { name: 'ACR_TARGET_REGISTRY', value: '${svcAcrName}${environment().suffixes.acrLoginServer}' }
             { name: 'TENANT_ID', value: tenant().tenantId }
             { name: 'DOCKER_CONFIG', value: '/auth' }
             { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId }
+            {
+              name: 'SECRETS'
+              value: '{"secrets":[{"registry": "quay.io", "azureSecretfile": "/auth/${pullSecretFile}"}]}'
+            }
           ]
         }
       ]

tooling/image-sync/README.md

@@ -10,18 +10,21 @@ The main configuration looks like this:
 repositories:
     - registry.k8s.io/external-dns/external-dns
 numberOfTags: 3
-quaySecretfile: /var/run/quay-secret.json
-acrRegistry: someregistry.azurecr.io
+acrTargetRegistry: someregistry.azurecr.io
 tenantId: 1ab61791-4b66-4ea4-85ff-aa2c0bf37e57
+secrets:
+  - registry: registry.k8s.io
+    secretFile: /secret.txt
 ```
 
 Explanation:
 - `repositories` - list of repositories to sync. Do not specify tags, since this utility will sync only the latest tags.
 - `numberOfTags` - number of tags to sync. The utility will sync the latest `numberOfTags` tags.
 - `quaySecretfile` - path to the secret file for the Quay registry.
-- `acrRegistry` - the target registry.
+- `acrTargetRegistry` - the target registry.
 - `tenantId` - the tenant ID used for authentication with Azure.
 - `RequestTimeout` - the timeout for the HTTP requests. Default is 10 seconds.
+- `secrets` - Array of secrets used for API authentitcation
 
 
 ### quaySecretfile

tooling/image-sync/example.yml

@@ -1,8 +1,9 @@
 repositories:
     - registry.k8s.io/external-dns/external-dns
-    - quay.io/acm-d/rhtap-hypershift-operator
-    - quay.io/app-sre/uhc-clusters-service
-numberOfTags: 40
-quaySecretfile: /home/jboll/workspace/opensource/quay-secret.json
-acrRegistry: testing.azurecr.io
+    - testingone.azurecr.io/azure-cli
+numberOfTags: 2
+acrTargetRegistry: testingtrgt.azurecr.io
 tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408
+secrets:
+  - registry: testingone.azurecr.io
+    azureSecretfile: /home/jboll/workspace/opensource/other-secret.json

tooling/image-sync/internal/repository.go

@@ -163,13 +163,13 @@ func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry {
 		}
 	}
 
-	client, err := azcontainerregistry.NewClient(fmt.Sprintf("https://%s", cfg.AcrRegistry), cred, nil)
+	client, err := azcontainerregistry.NewClient(fmt.Sprintf("https://%s", cfg.AcrTargetRegistry), cred, nil)
 	if err != nil {
 		Log().Fatalf("failed to create client: %v", err)
 	}
 
 	return &AzureContainerRegistry{
-		acrName:      cfg.AcrRegistry,
+		acrName:      cfg.AcrTargetRegistry,
 		acrClient:    client,
 		credential:   cred,
 		httpClient:   &http.Client{Timeout: time.Duration(cfg.RequestTimeout) * time.Second},
@@ -295,18 +295,124 @@ func (a *AzureContainerRegistry) GetTags(ctx context.Context, repository string)
 	return tags, nil
 }
 
+type ACRWithTokenAuth struct {
+	httpclient   *http.Client
+	acrName      string
+	numberOftags int
+	bearerToken  string
+}
+
+type AccessSecret struct {
+	AccessToken string `json:"access_token"`
+}
+
+type rawACRTagResponse struct {
+	Tags []rawACRTags
+}
+
+type rawACRTags struct {
+	Name string
+}
+
+func getACRBearerToken(ctx context.Context, secret AzureSecretFile, acrName string) (string, error) {
+	scope := "repository:*:*"
+	path := fmt.Sprintf("https://%s/oauth2/token?service=%s&scope=%s", acrName, acrName, scope)
+
+	Log().Debugw("Creating request", "path", path)
+	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
+	req.Header.Add("Authorization", fmt.Sprintf("Basic %s", secret.BasicAuthEncoded()))
+	if err != nil {
+		return "", fmt.Errorf("failed to create request: %v", err)
+	}
+
+	// todo replace with timeout enabled client
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to send request: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response: %v", err)
+	}
+
+	var accessSecret AccessSecret
+
+	err = json.Unmarshal(body, &accessSecret)
+	if err != nil {
+		return "", fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	return accessSecret.AccessToken, nil
+}
+
+func NewACRWithTokenAuth(cfg *SyncConfig, acrName string, bearerToken string) *ACRWithTokenAuth {
+	return &ACRWithTokenAuth{
+		httpclient:   &http.Client{Timeout: time.Duration(cfg.RequestTimeout) * time.Second},
+		acrName:      acrName,
+		bearerToken:  bearerToken,
+		numberOftags: cfg.NumberOfTags,
+	}
+}
+
+func (n *ACRWithTokenAuth) GetTags(ctx context.Context, image string) ([]string, error) {
+	Log().Debugw("Getting tags for image", "image", image)
+
+	path := fmt.Sprintf("https://%s/acr/v1/%s/_tags?orderby=%s&n=%d", n.acrName, image, azcontainerregistry.ArtifactTagOrderByLastUpdatedOnDescending, n.numberOftags)
+	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", n.bearerToken))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %v", err)
+	}
+
+	Log().Debugw("Sending request", "path", path)
+	resp, err := n.httpclient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %v", err)
+	}
+	Log().Debugw("Got response", "statuscode", resp.StatusCode)
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response: %v", err)
+	}
+
+	var acrResponse rawACRTagResponse
+	err = json.Unmarshal(body, &acrResponse)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	tagList := make([]string, 0)
+
+	for _, tag := range acrResponse.Tags {
+		tagList = append(tagList, tag.Name)
+	}
+
+	return tagList, nil
+}
+
 // OCIRegistry implements OCI Repository access
 type OCIRegistry struct {
 	httpclient   *http.Client
 	baseURL      string
 	numberOftags int
+	bearerToken  string
 }
 
 // NewOCIRegistry creates a new OCIRegistry access client
-func NewOCIRegistry(cfg *SyncConfig, baseURL string) *OCIRegistry {
+func NewOCIRegistry(cfg *SyncConfig, baseURL, bearerToken string) *OCIRegistry {
 	o := &OCIRegistry{
 		httpclient:   &http.Client{Timeout: time.Duration(cfg.RequestTimeout) * time.Second},
 		numberOftags: cfg.NumberOfTags,
+		bearerToken:  bearerToken,
 	}
 	if !strings.HasPrefix(o.baseURL, "https://") {
 		o.baseURL = fmt.Sprintf("https://%s", baseURL)
@@ -361,6 +467,9 @@ func (o *OCIRegistry) GetTags(ctx context.Context, image string) ([]string, erro
 
 	path := fmt.Sprintf("%s/v2/%s/tags/list", o.baseURL, image)
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
+	if o.bearerToken != "" {
+		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", o.bearerToken))
+	}
 	if err != nil {
 		return nil, fmt.Errorf("failed to create request: %v", err)
 	}

tooling/image-sync/internal/sync.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -22,19 +23,34 @@ func Log() *zap.SugaredLogger {
 type SyncConfig struct {
 	Repositories            []string
 	NumberOfTags            int
-	QuaySecretFile          string
-	AcrRegistry             string
+	Secrets                 []Secrets
+	AcrTargetRegistry       string
 	TenantId                string
 	RequestTimeout          int
 	AddLatest               bool
 	ManagedIdentityClientID string
 }
+type Secrets struct {
+	Registry        string
+	SecretFile      string
+	AzureSecretfile string
+}
 
-// QuaySecret is the secret for quay.io
-type QuaySecret struct {
+// BearerSecret is the secret for the source OCI registry
+type BearerSecret struct {
 	BearerToken string
 }
 
+// AzureSecret is the token configured in the ACR
+type AzureSecretFile struct {
+	Username string
+	Password string
+}
+
+func (a AzureSecretFile) BasicAuthEncoded() string {
+	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.Username, a.Password)))
+}
+
 // Copy copies an image from one registry to another
 func Copy(ctx context.Context, dstreference, srcreference string, dstauth, srcauth *types.DockerAuthConfig) error {
 	policyctx, err := signature.NewPolicyContext(&signature.Policy{
@@ -68,12 +84,26 @@ func Copy(ctx context.Context, dstreference, srcreference string, dstauth, srcau
 	return err
 }
 
-func readQuaySecret(filename string) (*QuaySecret, error) {
+func readBearerSecret(filename string) (*BearerSecret, error) {
+	secretBytes, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	var secret BearerSecret
+	err = json.Unmarshal(secretBytes, &secret)
+	if err != nil {
+		return nil, err
+	}
+
+	return &secret, nil
+}
+
+func readAzureSecret(filename string) (*AzureSecretFile, error) {
 	secretBytes, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, err
 	}
-	var secret QuaySecret
+	var secret AzureSecretFile
 	err = json.Unmarshal(secretBytes, &secret)
 	if err != nil {
 		return nil, err
@@ -103,19 +133,48 @@ func DoSync(cfg *SyncConfig) error {
 	Log().Infow("Syncing images", "images", cfg.Repositories, "numberoftags", cfg.NumberOfTags)
 	ctx := context.Background()
 
-	quaySecret, err := readQuaySecret(cfg.QuaySecretFile)
-	if err != nil {
-		return fmt.Errorf("error reading secret file: %w %s", err, cfg.QuaySecretFile)
+	srcRegistries := make(map[string]Registry)
+	var err error
+
+	for _, secret := range cfg.Secrets {
+		if secret.Registry == "quay.io" {
+			quaySecret, err := readBearerSecret(secret.SecretFile)
+			if err != nil {
+				return fmt.Errorf("error reading secret file: %w %s", err, secret.SecretFile)
+			}
+			qr := NewQuayRegistry(cfg, quaySecret.BearerToken)
+			srcRegistries[secret.Registry] = qr
+		} else {
+			if strings.HasSuffix(secret.Registry, "azurecr.io") ||
+				strings.HasSuffix(secret.Registry, "azurecr.cn") ||
+				strings.HasSuffix(secret.Registry, "azurecr.us") {
+				azureSecret, err := readAzureSecret(secret.AzureSecretfile)
+				if err != nil {
+					return fmt.Errorf("error reading azure secret file: %w %s", err, secret.AzureSecretfile)
+				}
+				bearerSecret, err := getACRBearerToken(ctx, *azureSecret, secret.Registry)
+				if err != nil {
+					return fmt.Errorf("error getting ACR bearer token: %w", err)
+				}
+				srcRegistries[secret.Registry] = NewACRWithTokenAuth(cfg, secret.Registry, bearerSecret)
+			} else {
+				s, err := readBearerSecret(secret.SecretFile)
+				bearerSecret := s.BearerToken
+				if err != nil {
+					return fmt.Errorf("error reading secret file: %w %s", err, secret.SecretFile)
+				}
+				srcRegistries[secret.Registry] = NewOCIRegistry(cfg, secret.Registry, bearerSecret)
+			}
+		}
 	}
-	qr := NewQuayRegistry(cfg, quaySecret.BearerToken)
 
-	acr := NewAzureContainerRegistry(cfg)
-	acrPullSecret, err := acr.GetPullSecret(ctx)
+	targetACR := NewAzureContainerRegistry(cfg)
+	acrPullSecret, err := targetACR.GetPullSecret(ctx)
 	if err != nil {
 		return fmt.Errorf("error getting pull secret: %w", err)
 	}
 
-	acrAuth := types.DockerAuthConfig{Username: "00000000-0000-0000-0000-000000000000", Password: acrPullSecret.RefreshToken}
+	targetACRAuth := types.DockerAuthConfig{Username: "00000000-0000-0000-0000-000000000000", Password: acrPullSecret.RefreshToken}
 
 	for _, repoName := range cfg.Repositories {
 		var srcTags, acrTags []string
@@ -125,28 +184,29 @@ func DoSync(cfg *SyncConfig) error {
 
 		Log().Infow("Syncing repository", "repository", repoName, "baseurl", baseURL)
 
-		if baseURL == "quay.io" {
-			srcTags, err = qr.GetTags(ctx, repoName)
+		if client, ok := srcRegistries[baseURL]; ok {
+			srcTags, err = client.GetTags(ctx, repoName)
 			if err != nil {
 				return fmt.Errorf("error getting quay tags: %w", err)
 			}
 			Log().Debugw("Got tags from quay", "tags", srcTags)
 		} else {
-			oci := NewOCIRegistry(cfg, baseURL)
+			// No secret defined, create a default client without auth
+			oci := NewOCIRegistry(cfg, baseURL, "")
 			srcTags, err = oci.GetTags(ctx, repoName)
 			if err != nil {
 				return fmt.Errorf("error getting oci tags: %w", err)
 			}
 			Log().Debugw(fmt.Sprintf("Got tags from %s", baseURL), "repo", repoName, "tags", srcTags)
 		}
 
-		exists, err := acr.RepositoryExists(ctx, repoName)
+		exists, err := targetACR.RepositoryExists(ctx, repoName)
 		if err != nil {
 			return fmt.Errorf("error getting ACR repository information: %w", err)
 		}
 
 		if exists {
-			acrTags, err = acr.GetTags(ctx, repoName)
+			acrTags, err = targetACR.GetTags(ctx, repoName)
 			if err != nil {
 				return fmt.Errorf("error getting ACR tags: %w", err)
 			}
@@ -161,10 +221,10 @@ func DoSync(cfg *SyncConfig) error {
 
 		for _, tagToSync := range tagsToSync {
 			source := fmt.Sprintf("%s/%s:%s", baseURL, repoName, tagToSync)
-			target := fmt.Sprintf("%s/%s:%s", cfg.AcrRegistry, repoName, tagToSync)
+			target := fmt.Sprintf("%s/%s:%s", cfg.AcrTargetRegistry, repoName, tagToSync)
 			Log().Infow("Copying images", "images", tagToSync, "from", source, "to", target)
 
-			err = Copy(ctx, target, source, &acrAuth, nil)
+			err = Copy(ctx, target, source, &targetACRAuth, nil)
 			if err != nil {
 				return fmt.Errorf("error copying image: %w", err)
 			}