From 3a291e368aa9c189d0f906d40c7288f5ef336119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Mon, 19 Jun 2023 08:37:58 +0800 Subject: [PATCH 1/2] Make Kopia support Azure AD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit introduces our own Azure storage provider by wrapping Kopia's implementation rather than contributing to upstream based on the following considerations: 1. Velero needs the capability to interact with the repository concurrently while Kopia doesn't, this will increase the complexity of Kopia if we contribute to upstream 2. The configuration items provided by Velero and Kopia are conflict, e.g. Velero supports customizing storage account URI which is a full path while Kopia supports customizing storage account domain which is part of the URI. We need to consider the backward compatibility and upgrade case if we contribute to upstream which needs extra efforts 3. Contribute to upstream is a longer cycle when we need to introduce new changes. With this commit, we no longer depends on upstream for the Azure storage provider part and is easy for us to maintain Signed-off-by: Wenkai Yin(尹文开) --- changelogs/unreleased/6686-ywk253100 | 1 + go.mod | 9 +- go.sum | 9 +- pkg/repository/config/azure.go | 219 +------------- pkg/repository/config/azure_test.go | 181 ++---------- pkg/repository/provider/unified_repo.go | 19 +- pkg/repository/provider/unified_repo_test.go | 140 ++------- .../udmrepo/kopialib/backend/azure.go | 31 +- .../backend/azure/azure_storage_wrapper.go | 78 +++++ .../udmrepo/kopialib/backend/azure_test.go | 85 +----- pkg/repository/udmrepo/repo_options.go | 5 - pkg/util/azure/credential.go | 133 +++++++++ pkg/util/azure/credential_test.go | 96 ++++++ pkg/util/azure/storage.go | 276 ++++++++++++++++++ pkg/util/azure/storage_test.go | 223 ++++++++++++++ pkg/util/azure/testdata/certificate.pem | 49 ++++ pkg/util/azure/util.go | 109 +++++++ pkg/util/azure/util_test.go | 199 +++++++++++++ 18 files changed, 1269 insertions(+), 593 deletions(-) create mode 100644 changelogs/unreleased/6686-ywk253100 create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go create mode 100644 pkg/util/azure/credential.go create mode 100644 pkg/util/azure/credential_test.go create mode 100644 pkg/util/azure/storage.go create mode 100644 pkg/util/azure/storage_test.go create mode 100644 pkg/util/azure/testdata/certificate.pem create mode 100644 pkg/util/azure/util.go create mode 100644 pkg/util/azure/util_test.go diff --git a/changelogs/unreleased/6686-ywk253100 b/changelogs/unreleased/6686-ywk253100 new file mode 100644 index 0000000000..d2a79be6db --- /dev/null +++ b/changelogs/unreleased/6686-ywk253100 @@ -0,0 +1 @@ +Make Kopia support Azure AD \ No newline at end of file diff --git a/go.mod b/go.mod index 4a6f861854..51cd719132 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,15 @@ require ( cloud.google.com/go/storage v1.32.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-sdk-for-go v67.2.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.3.0 - github.com/aws/aws-sdk-go v1.44.253 + github.com/aws/aws-sdk-go v1.44.256 github.com/bombsimon/logrusr/v3 v3.0.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/fatih/color v1.15.0 @@ -62,10 +66,7 @@ require ( cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect diff --git a/go.sum b/go.sum index d89c868ae0..bc4c0ed7c0 100644 --- a/go.sum +++ b/go.sum @@ -58,7 +58,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 h1:LcJtQjCXJUm1s7JpUHZvu+bpgURhCatxVNbGADXniX0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0/go.mod h1:+OgGVo0Httq7N5oayfvaLQ/Jq+2gJdqfp++Hyyl7Tws= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= @@ -133,8 +136,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.44.253 h1:iqDd0okcH4ShfFexz2zzf4VmeDFf6NOMm07pHnEb8iY= -github.com/aws/aws-sdk-go v1.44.253/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= diff --git a/pkg/repository/config/azure.go b/pkg/repository/config/azure.go index 1c203330e1..6662d13c6d 100644 --- a/pkg/repository/config/azure.go +++ b/pkg/repository/config/azure.go @@ -17,225 +17,34 @@ limitations under the License. package config import ( - "context" - "fmt" - "os" - "strings" - - storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" - "github.com/Azure/go-autorest/autorest/azure" - "github.com/Azure/go-autorest/autorest/azure/auth" - "github.com/joho/godotenv" "github.com/pkg/errors" -) - -const ( - subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID" - cloudNameEnvVar = "AZURE_CLOUD_NAME" - resourceGroupConfigKey = "resourceGroup" - - storageAccountConfigKey = "storageAccount" - storageAccountKeyEnvVarConfigKey = "storageAccountKeyEnvVar" - subscriptionIDConfigKey = "subscriptionId" - storageDomainConfigKey = "storageDomain" + "github.com/vmware-tanzu/velero/pkg/util/azure" ) -// getSubscriptionID gets the subscription ID from the 'config' map if it contains -// it, else from the AZURE_SUBSCRIPTION_ID environment variable. -func getSubscriptionID(config map[string]string) string { - if subscriptionID := config[subscriptionIDConfigKey]; subscriptionID != "" { - return subscriptionID - } - - return os.Getenv(subscriptionIDEnvVar) -} - -func getStorageAccountKey(config map[string]string) (string, error) { - credentialsFile := selectCredentialsFile(config) - - if err := loadCredentialsIntoEnv(credentialsFile); err != nil { - return "", err - } - - // Get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not - // exist, parseAzureEnvironment will return azure.PublicCloud. - env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar)) - if err != nil { - return "", errors.Wrap(err, "unable to parse azure cloud name environment variable") - } - - // Get storage key from secret using key config[storageAccountKeyEnvVarConfigKey]. If the config does not - // exist, continue obtaining it using API - if secretKeyEnvVar := config[storageAccountKeyEnvVarConfigKey]; secretKeyEnvVar != "" { - storageKey := os.Getenv(secretKeyEnvVar) - if storageKey == "" { - return "", errors.Errorf("no storage key secret with key %s found", secretKeyEnvVar) - } - - return storageKey, nil - } - - // get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable - subscriptionID := getSubscriptionID(config) - if subscriptionID == "" { - return "", errors.New("azure subscription ID not found in object store's config or in environment variable") - } - - // we need config["resourceGroup"], config["storageAccount"] - if err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil { - return "", errors.Wrap(err, "unable to get all required config values") - } - - // get authorizer from environment in the following order: - // 1. client credentials (AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET) - // 2. client certificate (AZURE_CERTIFICATE_PATH, AZURE_CERTIFICATE_PASSWORD) - // 3. username and password (AZURE_USERNAME, AZURE_PASSWORD) - // 4. MSI (managed service identity) - authorizer, err := auth.NewAuthorizerFromEnvironment() - if err != nil { - return "", errors.Wrap(err, "error getting authorizer from environment") - } - - // get storageAccountsClient - storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID) - storageAccountsClient.Authorizer = authorizer - - // get storage key - res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey], storagemgmt.Kerb) - if err != nil { - return "", errors.WithStack(err) - } - if res.Keys == nil || len(*res.Keys) == 0 { - return "", errors.New("No storage keys found") - } - - var storageKey string - for _, key := range *res.Keys { - // The ListKeys call returns e.g. "FULL" but the storagemgmt.Full constant in the SDK is defined as "Full". - if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) { - storageKey = *key.Value - break - } - } - - if storageKey == "" { - return "", errors.New("No storage key with Full permissions found") - } - - return storageKey, nil -} - -func mapLookup(data map[string]string) func(string) string { - return func(key string) string { - return data[key] - } -} - // GetAzureResticEnvVars gets the environment variables that restic // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based // on info in the provided object storage location config map. func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) { - storageAccountKey, err := getStorageAccountKey(config) - if err != nil { - return nil, err + storageAccount := config[azure.BSLConfigStorageAccount] + if storageAccount == "" { + return nil, errors.New("storageAccount is required in the BSL") } - if err := getRequiredValues(mapLookup(config), storageAccountConfigKey); err != nil { - return nil, errors.Wrap(err, "unable to get all required config values") - } - - return map[string]string{ - "AZURE_ACCOUNT_NAME": config[storageAccountConfigKey], - "AZURE_ACCOUNT_KEY": storageAccountKey, - }, nil -} - -// credentialsFileFromEnv retrieves the Azure credentials file from the environment. -func credentialsFileFromEnv() string { - return os.Getenv("AZURE_CREDENTIALS_FILE") -} - -// selectCredentialsFile selects the Azure credentials file to use, retrieving it -// from the given config or falling back to retrieving it from the environment. -func selectCredentialsFile(config map[string]string) string { - if credentialsFile, ok := config[CredentialsFileKey]; ok { - return credentialsFile - } - - return credentialsFileFromEnv() -} - -// loadCredentialsIntoEnv loads the variables in the given credentials -// file into the current environment. -func loadCredentialsIntoEnv(credentialsFile string) error { - if credentialsFile == "" { - return nil - } - - if err := godotenv.Overload(credentialsFile); err != nil { - return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) - } - - return nil -} - -// ParseAzureEnvironment returns an azure.Environment for the given cloud -// name, or azure.PublicCloud if cloudName is empty. -func parseAzureEnvironment(cloudName string) (*azure.Environment, error) { - if cloudName == "" { - return &azure.PublicCloud, nil - } - - env, err := azure.EnvironmentFromName(cloudName) - return &env, errors.WithStack(err) -} - -func getRequiredValues(getValue func(string) string, keys ...string) error { - missing := []string{} - results := map[string]string{} - - for _, key := range keys { - if val := getValue(key); val == "" { - missing = append(missing, key) - } else { - results[key] = val - } - } - - if len(missing) > 0 { - return errors.Errorf("the following keys do not have values: %s", strings.Join(missing, ", ")) - } - - return nil -} - -// GetAzureStorageDomain gets the Azure storage domain required by a Azure blob connection, -// if the provided credential file doesn't have the value, get it from system's environment variables -func GetAzureStorageDomain(config map[string]string) (string, error) { - credentialsFile := selectCredentialsFile(config) - - if err := loadCredentialsIntoEnv(credentialsFile); err != nil { - return "", err - } - - return getStorageDomainFromCloudName(os.Getenv(cloudNameEnvVar)) -} - -func GetAzureCredentials(config map[string]string) (string, string, error) { - storageAccountKey, err := getStorageAccountKey(config) + creds, err := azure.LoadCredentials(config) if err != nil { - return "", "", err + return nil, err } - return config[storageAccountConfigKey], storageAccountKey, nil -} - -func getStorageDomainFromCloudName(cloudName string) (string, error) { - env, err := parseAzureEnvironment(cloudName) + // restic doesn't support Azure AD, set it as false + config[azure.BSLConfigUseAAD] = "false" + credentials, err := azure.GetStorageAccountCredentials(config, creds) if err != nil { - return "", errors.Wrapf(err, "unable to parse azure env from cloud name %s", cloudName) + return nil, err } - return fmt.Sprintf("blob.%s", env.StorageEndpointSuffix), nil + return map[string]string{ + "AZURE_ACCOUNT_NAME": storageAccount, + "AZURE_ACCOUNT_KEY": credentials[azure.CredentialKeyStorageAccountAccessKey], + }, nil } diff --git a/pkg/repository/config/azure_test.go b/pkg/repository/config/azure_test.go index 87cd31bf15..c283197b3a 100644 --- a/pkg/repository/config/azure_test.go +++ b/pkg/repository/config/azure_test.go @@ -1,12 +1,9 @@ /* Copyright the Velero contributors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,161 +15,37 @@ package config import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" -) - -// setAzureEnvironment sets the Azure credentials environment variable to the -// given value and returns a function to restore it to its previous value -func setAzureEnvironment(t *testing.T, value string) func() { - envVar := "AZURE_CREDENTIALS_FILE" - var cleanup func() - - if original, exists := os.LookupEnv(envVar); exists { - cleanup = func() { - require.NoError(t, os.Setenv(envVar, original), "failed to reset %s environment variable", envVar) - } - } else { - cleanup = func() { - require.NoError(t, os.Unsetenv(envVar), "failed to reset %s environment variable", envVar) - } - } - - require.NoError(t, os.Setenv(envVar, value), "failed to set %s environment variable", envVar) - - return cleanup -} -func TestSelectCredentialsFile(t *testing.T) { - testCases := []struct { - name string - config map[string]string - environment string - expected string - }{ - { - name: "when config is empty and environment variable is not set, no file is selected", - expected: "", - }, - { - name: "when config contains credentials file and environment variable is not set, file from config is selected", - config: map[string]string{ - "credentialsFile": "/tmp/credentials/path/to/secret", - }, - expected: "/tmp/credentials/path/to/secret", - }, - { - name: "when config is empty and environment variable is set, file from environment is selected", - environment: "/credentials/file/from/env", - expected: "/credentials/file/from/env", - }, - { - name: "when config contains credentials file and environment variable is set, file from config is selected", - config: map[string]string{ - "credentialsFile": "/tmp/credentials/path/to/secret", - }, - environment: "/credentials/file/from/env", - expected: "/tmp/credentials/path/to/secret", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - cleanup := setAzureEnvironment(t, tc.environment) - defer cleanup() - - selectedFile := selectCredentialsFile(tc.config) - require.Equal(t, tc.expected, selectedFile) - }) - } -} - -func TestGetStorageDomainFromCloudName(t *testing.T) { - testCases := []struct { - name string - cloudName string - expected string - expectedErr string - }{ - { - name: "get azure env fail", - cloudName: "fake-cloud", - expectedErr: "unable to parse azure env from cloud name fake-cloud: autorest/azure: There is no cloud environment matching the name \"FAKE-CLOUD\"", - }, - { - name: "cloud name is empty", - cloudName: "", - expected: "blob.core.windows.net", - }, - { - name: "azure public cloud", - cloudName: "AzurePublicCloud", - expected: "blob.core.windows.net", - }, - { - - name: "azure China cloud", - cloudName: "AzureChinaCloud", - expected: "blob.core.chinacloudapi.cn", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - domain, err := getStorageDomainFromCloudName(tc.cloudName) - - require.Equal(t, tc.expected, domain) - - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.expectedErr) - assert.Empty(t, domain) - } - }) - } -} - -func TestGetRequiredValues(t *testing.T) { - testCases := []struct { - name string - mp map[string]string - keys []string - err string - }{ - { - name: "with miss", - mp: map[string]string{ - "key1": "value1", - }, - keys: []string{"key1", "key2", "key3"}, - err: "the following keys do not have values: key2, key3", - }, - { - name: "without miss", - mp: map[string]string{ - "key1": "value1", - "key2": "value2", - "key3": "value3", - }, - keys: []string{"key1", "key2", "key3"}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := getRequiredValues(func(key string) string { - if tc.mp == nil { - return "" - } else { - return tc.mp[key] - } - }, tc.keys...) + "github.com/vmware-tanzu/velero/pkg/util/azure" +) - if err == nil { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.err) - } - }) - } +func TestGetAzureResticEnvVars(t *testing.T) { + config := map[string]string{} + + // no storage account specified + _, err := GetAzureResticEnvVars(config) + require.NotNil(t, err) + + // specify storage account access key + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("AccessKey: accesskey") + require.Nil(t, err) + + config[azure.BSLConfigStorageAccount] = "account01" + config[azure.BSLConfigStorageAccountAccessKeyName] = "AccessKey" + config["credentialsFile"] = name + envs, err := GetAzureResticEnvVars(config) + require.Nil(t, err) + + assert.Equal(t, "account01", envs["AZURE_ACCOUNT_NAME"]) + assert.Equal(t, "accesskey", envs["AZURE_ACCOUNT_KEY"]) } diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 8dc4701855..f684cc4986 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -47,11 +47,9 @@ type unifiedRepoProvider struct { // this func is assigned to a package-level variable so it can be // replaced when unit-testing -var getAzureCredentials = repoconfig.GetAzureCredentials var getS3Credentials = repoconfig.GetS3Credentials var getGCPCredentials = repoconfig.GetGCPCredentials var getS3BucketRegion = repoconfig.GetAWSBucketRegion -var getAzureStorageDomain = repoconfig.GetAzureStorageDomain type localFuncTable struct { getStorageVariables func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) @@ -190,6 +188,7 @@ func (urp *unifiedRepoProvider) PrepareRepo(ctx context.Context, param RepoParam log.Debug("Repo has already been initialized remotely") return nil } + log.Infof("failed to connect to the repo: %v, will try to create it", err) err = urp.repoService.Init(ctx, *repoOption, true) if err != nil { @@ -436,13 +435,8 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr result[udmrepo.StoreOptionS3Token] = credValue.SessionToken } case repoconfig.AzureBackend: - storageAccount, accountKey, err := getAzureCredentials(config) - if err != nil { - return map[string]string{}, errors.Wrap(err, "error get azure credentials") - } - result[udmrepo.StoreOptionAzureStorageAccount] = storageAccount - result[udmrepo.StoreOptionAzureKey] = accountKey - + // do nothing here, will retrieve the credential in Azure Storage + return nil, nil case repoconfig.GCPBackend: result[udmrepo.StoreOptionCredentialFile] = getGCPCredentials(config) } @@ -509,12 +503,9 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo result[udmrepo.StoreOptionS3CustomCA] = base64.StdEncoding.EncodeToString(backupLocation.Spec.ObjectStorage.CACert) } } else if backendType == repoconfig.AzureBackend { - domain, err := getAzureStorageDomain(config) - if err != nil { - return map[string]string{}, errors.Wrapf(err, "error to get azure storage domain") + for k, v := range config { + result[k] = v } - - result[udmrepo.StoreOptionAzureDomain] = domain } result[udmrepo.StoreOptionOssBucket] = bucket diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 08f33eb575..0910ef841c 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -39,16 +39,15 @@ import ( func TestGetStorageCredentials(t *testing.T) { testCases := []struct { - name string - backupLocation velerov1api.BackupStorageLocation - credFileStore *credmock.FileStore - credStoreError error - credStorePath string - getAzureCredentials func(map[string]string) (string, string, error) - getS3Credentials func(map[string]string) (*awscredentials.Value, error) - getGCPCredentials func(map[string]string) string - expected map[string]string - expectedErr string + name string + backupLocation velerov1api.BackupStorageLocation + credFileStore *credmock.FileStore + credStoreError error + credStorePath string + getS3Credentials func(map[string]string) (*awscredentials.Value, error) + getGCPCredentials func(map[string]string) string + expected map[string]string + expectedErr string }{ { name: "invalid credentials file store interface", @@ -160,43 +159,15 @@ func TestGetStorageCredentials(t *testing.T) { expected: map[string]string{}, }, { - name: "azure, Credential section exists in BSL", + name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", - }, + Provider: "velero.io/azure", Credential: &corev1api.SecretKeySelector{}, }, }, credFileStore: new(credmock.FileStore), - credStorePath: "credentials-from-credential-key", - getAzureCredentials: func(config map[string]string) (string, string, error) { - return "storage account from: " + config["credentialsFile"], "", nil - }, - - expected: map[string]string{ - "storageAccount": "storage account from: credentials-from-credential-key", - "storageKey": "", - }, - }, - { - name: "azure, get azure credentials fails", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", - }, - }, - }, - getAzureCredentials: func(config map[string]string) (string, string, error) { - return "", "", errors.New("fake error") - }, - credFileStore: new(credmock.FileStore), - expected: map[string]string{}, - expectedErr: "error get azure credentials: fake error", + expected: nil, }, { name: "gcp, Credential section not exists in BSL", @@ -220,7 +191,6 @@ func TestGetStorageCredentials(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - getAzureCredentials = tc.getAzureCredentials getS3Credentials = tc.getS3Credentials getGCPCredentials = tc.getGCPCredentials @@ -245,14 +215,14 @@ func TestGetStorageCredentials(t *testing.T) { func TestGetStorageVariables(t *testing.T) { testCases := []struct { - name string - backupLocation velerov1api.BackupStorageLocation - repoName string - repoBackend string - getS3BucketRegion func(string) (string, error) - getAzureStorageDomain func(map[string]string) (string, error) - expected map[string]string - expectedErr string + name string + backupLocation velerov1api.BackupStorageLocation + credFileStore *credmock.FileStore + repoName string + repoBackend string + getS3BucketRegion func(string) (string, error) + expected map[string]string + expectedErr string }{ { name: "invalid provider", @@ -418,7 +388,7 @@ func TestGetStorageVariables(t *testing.T) { }, }, { - name: "azure, getAzureStorageDomain fail", + name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/azure", @@ -436,68 +406,13 @@ func TestGetStorageVariables(t *testing.T) { }, }, }, - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "", errors.New("fake error") - }, - repoBackend: "fake-repo-type", - expected: map[string]string{}, - expectedErr: "error to get azure storage domain: fake error", - }, - { - name: "azure, ObjectStorage section exists in BSL", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "bucket": "fake-bucket-config", - "prefix": "fake-prefix-config", - "region": "fake-region", - "fspath": "", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "fake-bucket-object-store", - Prefix: "fake-prefix-object-store", - }, - }, - }, - }, - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "fake-domain", nil - }, - repoBackend: "fake-repo-type", - expected: map[string]string{ - "bucket": "fake-bucket-object-store", - "prefix": "fake-prefix-object-store/fake-repo-type/", - "region": "fake-region", - "fspath": "", - "storageDomain": "fake-domain", - }, - }, - { - name: "azure, ObjectStorage section not exists in BSL, repo name exists", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "bucket": "fake-bucket", - "prefix": "fake-prefix", - "region": "fake-region", - "fspath": "", - }, - }, - }, - repoName: "//fake-name//", - repoBackend: "fake-repo-type", - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "fake-domain", nil - }, + credFileStore: new(credmock.FileStore), + repoBackend: "fake-repo-type", expected: map[string]string{ - "bucket": "fake-bucket", - "prefix": "fake-prefix/fake-repo-type/fake-name/", - "region": "fake-region", - "fspath": "", - "storageDomain": "fake-domain", + "bucket": "fake-bucket-object-store", + "prefix": "fake-prefix-object-store/fake-repo-type/", + "region": "fake-region", + "fspath": "", }, }, { @@ -524,7 +439,6 @@ func TestGetStorageVariables(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { getS3BucketRegion = tc.getS3BucketRegion - getAzureStorageDomain = tc.getAzureStorageDomain actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName) diff --git a/pkg/repository/udmrepo/kopialib/backend/azure.go b/pkg/repository/udmrepo/kopialib/backend/azure.go index d5243820f4..ddf0e4d925 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure.go @@ -20,41 +20,22 @@ import ( "context" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/blob/azure" - "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure" ) type AzureBackend struct { - options azure.Options + option azure.Option } func (c *AzureBackend) Setup(ctx context.Context, flags map[string]string) error { - var err error - c.options.Container, err = mustHaveString(udmrepo.StoreOptionOssBucket, flags) - if err != nil { - return err + c.option = azure.Option{ + Config: flags, + Limits: setupLimits(ctx, flags), } - - c.options.StorageAccount, err = mustHaveString(udmrepo.StoreOptionAzureStorageAccount, flags) - if err != nil { - return err - } - - c.options.StorageKey, err = mustHaveString(udmrepo.StoreOptionAzureKey, flags) - if err != nil { - return err - } - - c.options.Prefix = optionalHaveString(udmrepo.StoreOptionPrefix, flags) - c.options.SASToken = optionalHaveString(udmrepo.StoreOptionAzureToken, flags) - c.options.StorageDomain = optionalHaveString(udmrepo.StoreOptionAzureDomain, flags) - - c.options.Limits = setupLimits(ctx, flags) - return nil } func (c *AzureBackend) Connect(ctx context.Context, isCreate bool) (blob.Storage, error) { - return azure.New(ctx, &c.options, false) + return azure.NewStorage(ctx, &c.option, false) } diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go new file mode 100644 index 0000000000..5e855611c2 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go @@ -0,0 +1,78 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/blob/throttling" + "github.com/sirupsen/logrus" + + "github.com/kopia/kopia/repo/blob/azure" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + azureutil "github.com/vmware-tanzu/velero/pkg/util/azure" +) + +const ( + storageType = "azure" +) + +func init() { + blob.AddSupportedStorage(storageType, Option{}, NewStorage) +} + +type Option struct { + Config map[string]string `json:"config" kopia:"sensitive"` + Limits throttling.Limits +} + +type Storage struct { + blob.Storage + Option *Option +} + +func (s *Storage) ConnectionInfo() blob.ConnectionInfo { + return blob.ConnectionInfo{ + Type: storageType, + Config: s.Option, + } +} + +func NewStorage(ctx context.Context, option *Option, isCreate bool) (blob.Storage, error) { + cfg := option.Config + + client, _, err := azureutil.NewStorageClient(logrus.New(), cfg) + if err != nil { + return nil, err + } + + opt := &azure.Options{ + Container: cfg[udmrepo.StoreOptionOssBucket], + Prefix: cfg[udmrepo.StoreOptionPrefix], + Limits: option.Limits, + } + azStorage, err := azure.NewWithClient(ctx, opt, client) + if err != nil { + return nil, err + } + + return &Storage{ + Option: option, + Storage: azStorage, + }, nil +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure_test.go b/pkg/repository/udmrepo/kopialib/backend/azure_test.go index bc4997fbe7..6814c635ab 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure_test.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure_test.go @@ -20,83 +20,28 @@ import ( "context" "testing" + "github.com/kopia/kopia/repo/blob/throttling" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" - - "github.com/kopia/kopia/repo/blob/azure" - "github.com/kopia/kopia/repo/blob/throttling" ) func TestAzureSetup(t *testing.T) { - testCases := []struct { - name string - flags map[string]string - expected azure.Options - expectedErr string - }{ - { - name: "must have bucket name", - flags: map[string]string{}, - expectedErr: "key " + udmrepo.StoreOptionOssBucket + " not found", - }, - { - name: "must have storage account", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - }, - expected: azure.Options{ - Container: "fake-bucket", - }, - expectedErr: "key " + udmrepo.StoreOptionAzureStorageAccount + " not found", - }, - { - name: "must have secret key", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionAzureStorageAccount: "fake-account", - }, - expected: azure.Options{ - Container: "fake-bucket", - StorageAccount: "fake-account", - }, - expectedErr: "key " + udmrepo.StoreOptionAzureKey + " not found", - }, - { - name: "with limits", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionAzureStorageAccount: "fake-account", - udmrepo.StoreOptionAzureKey: "fake-key", - udmrepo.ThrottleOptionReadOps: "100", - udmrepo.ThrottleOptionUploadBytes: "200", - }, - expected: azure.Options{ - Container: "fake-bucket", - StorageAccount: "fake-account", - StorageKey: "fake-key", - Limits: throttling.Limits{ - ReadsPerSecond: 100, - UploadBytesPerSecond: 200, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - azFlags := AzureBackend{} + backend := AzureBackend{} - err := azFlags.Setup(context.Background(), tc.flags) - - require.Equal(t, tc.expected, azFlags.options) - - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.expectedErr) - } - }) + flags := map[string]string{ + "key": "value", + udmrepo.ThrottleOptionReadOps: "100", + udmrepo.ThrottleOptionUploadBytes: "200", } + limits := throttling.Limits{ + ReadsPerSecond: 100, + UploadBytesPerSecond: 200, + } + + err := backend.Setup(context.Background(), flags) + require.Nil(t, err) + assert.Equal(t, flags, backend.option.Config) + assert.Equal(t, limits, backend.option.Limits) } diff --git a/pkg/repository/udmrepo/repo_options.go b/pkg/repository/udmrepo/repo_options.go index 9337018f23..09acc49f72 100644 --- a/pkg/repository/udmrepo/repo_options.go +++ b/pkg/repository/udmrepo/repo_options.go @@ -44,11 +44,6 @@ const ( StoreOptionS3DisableTLSVerify = "skipTLSVerify" StoreOptionS3CustomCA = "customCA" - StoreOptionAzureKey = "storageKey" - StoreOptionAzureDomain = "storageDomain" - StoreOptionAzureStorageAccount = "storageAccount" - StoreOptionAzureToken = "sasToken" - StoreOptionFsPath = "fspath" StoreOptionGcsReadonly = "readonly" diff --git a/pkg/util/azure/credential.go b/pkg/util/azure/credential.go new file mode 100644 index 0000000000..6d5e16ded3 --- /dev/null +++ b/pkg/util/azure/credential.go @@ -0,0 +1,133 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/pkg/errors" +) + +// NewCredential chains the config credential and workload identity credential +func NewCredential(creds map[string]string, options policy.ClientOptions) (azcore.TokenCredential, error) { + var ( + credential []azcore.TokenCredential + errMsgs []string + ) + + additionalTenants := []string{} + if tenants := creds[CredentialKeyAdditionallyAllowedTenants]; tenants != "" { + additionalTenants = strings.Split(tenants, ";") + } + + // config credential + cfgCred, err := newConfigCredential(creds, configCredentialOptions{ + ClientOptions: options, + AdditionallyAllowedTenants: additionalTenants, + }) + if err == nil { + credential = append(credential, cfgCred) + } else { + errMsgs = append(errMsgs, err.Error()) + } + + // workload identity credential + wic, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + AdditionallyAllowedTenants: additionalTenants, + ClientOptions: options, + }) + if err == nil { + credential = append(credential, wic) + } else { + errMsgs = append(errMsgs, err.Error()) + } + + if len(credential) == 0 { + return nil, errors.Errorf("failed to create Azure credential: %s", strings.Join(errMsgs, "\n\t")) + } + + return azidentity.NewChainedTokenCredential(credential, nil) +} + +type configCredentialOptions struct { + azcore.ClientOptions + AdditionallyAllowedTenants []string +} + +// newConfigCredential works same as the azidentity.EnvironmentCredential but reads the credentials from a map +// rather than environment variables. This is required for Velero to run B/R concurrently +// https://github.com/Azure/azure-sdk-for-go/blob/sdk/azidentity/v1.3.0/sdk/azidentity/environment_credential.go#L80 +func newConfigCredential(creds map[string]string, options configCredentialOptions) (azcore.TokenCredential, error) { + tenantID := creds[CredentialKeyTenantID] + if tenantID == "" { + return nil, errors.Errorf("%s is required", CredentialKeyTenantID) + } + clientID := creds[CredentialKeyClientID] + if clientID == "" { + return nil, errors.Errorf("%s is required", CredentialKeyClientID) + } + + // client secret + if clientSecret := creds[CredentialKeyClientSecret]; clientSecret != "" { + return azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, &azidentity.ClientSecretCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + }) + } + + // certificate + if certPath := creds[CredentialKeyClientCertificatePath]; certPath != "" { + certData, err := os.ReadFile(certPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read certificate file %s", certPath) + } + var password []byte + if v := creds[CredentialKeyClientCertificatePassword]; v != "" { + password = []byte(v) + } + certs, key, err := azidentity.ParseCertificates(certData, password) + if err != nil { + return nil, errors.Wrapf(err, "failed to load certificate from %s", certPath) + } + o := &azidentity.ClientCertificateCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + } + if v, ok := creds[CredentialKeySendCertChain]; ok { + o.SendCertificateChain = v == "1" || strings.ToLower(v) == "true" + } + return azidentity.NewClientCertificateCredential(tenantID, clientID, certs, key, o) + } + + // username/password + if username := creds[CredentialKeyUsername]; username != "" { + if password := creds[CredentialKeyPassword]; password != "" { + return azidentity.NewUsernamePasswordCredential(tenantID, clientID, username, password, + &azidentity.UsernamePasswordCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + }) + } + return nil, errors.Errorf("%s is required", CredentialKeyPassword) + } + + return nil, errors.New("incomplete credential configuration. Only AZURE_TENANT_ID and AZURE_CLIENT_ID are set") +} diff --git a/pkg/util/azure/credential_test.go b/pkg/util/azure/credential_test.go new file mode 100644 index 0000000000..4d6d7f0d33 --- /dev/null +++ b/pkg/util/azure/credential_test.go @@ -0,0 +1,96 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +func TestNewCredential(t *testing.T) { + options := policy.ClientOptions{} + + // no credentials + creds := map[string]string{} + _, err := NewCredential(creds, options) + require.NotNil(t, err) + + // config credential + creds = map[string]string{ + CredentialKeyTenantID: "tenantid", + CredentialKeyClientID: "clientid", + CredentialKeyClientSecret: "secret", + } + _, err = NewCredential(creds, options) + require.Nil(t, err) +} + +func Test_newConfigCredential(t *testing.T) { + options := configCredentialOptions{} + + // tenantID not specified + creds := map[string]string{} + _, err := newConfigCredential(creds, options) + require.NotNil(t, err) + + // clientID not specified + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + } + _, err = newConfigCredential(creds, options) + require.NotNil(t, err) + + // client secret + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyClientSecret: "secret", + } + credential, err := newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok := credential.(*azidentity.ClientSecretCredential) + require.True(t, ok) + + // client certificate + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyClientCertificatePath: "testdata/certificate.pem", + } + credential, err = newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok = credential.(*azidentity.ClientCertificateCredential) + require.True(t, ok) + + // username/password + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyUsername: "username", + CredentialKeyPassword: "password", + } + credential, err = newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok = credential.(*azidentity.UsernamePasswordCredential) + require.True(t, ok) +} diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go new file mode 100644 index 0000000000..49943a3f92 --- /dev/null +++ b/pkg/util/azure/storage.go @@ -0,0 +1,276 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + _ "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + // the keys of Azure BSL config: + // https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md + BSLConfigResourceGroup = "resourceGroup" + BSLConfigStorageAccount = "storageAccount" + BSLConfigStorageAccountAccessKeyName = "storageAccountKeyEnvVar" + BSLConfigSubscriptionID = "subscriptionId" + BSLConfigStorageAccountURI = "storageAccountURI" + BSLConfigUseAAD = "useAAD" + BSLConfigActiveDirectoryAuthorityURI = "activeDirectoryAuthorityURI" + + serviceNameBlob cloud.ServiceName = "blob" +) + +func init() { + cloud.AzureChina.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.chinacloudapi.cn", + } + cloud.AzureGovernment.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.usgovcloudapi.net", + } + cloud.AzurePublic.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.windows.net", + } +} + +// NewStorageClient creates a blob storage client(data plane) with the provided config which contains BSL config and the credential file name. +// The returned azblob.SharedKeyCredential is needed for Azure plugin to generate the SAS URL when auth with storage +// account access key +func NewStorageClient(log logrus.FieldLogger, config map[string]string) (*azblob.Client, *azblob.SharedKeyCredential, error) { + // rename to bslCfg for easy understanding + bslCfg := config + + // storage account is required + storageAccount := bslCfg[BSLConfigStorageAccount] + if storageAccount == "" { + return nil, nil, errors.Errorf("%s is required in BSL", BSLConfigStorageAccount) + } + + // read the credentials provided by users + creds, err := LoadCredentials(config) + if err != nil { + return nil, nil, err + } + // exchange the storage account access key if needed + creds, err = GetStorageAccountCredentials(bslCfg, creds) + if err != nil { + return nil, nil, err + } + + // get the storage account URI + uri, err := getStorageAccountURI(log, bslCfg, creds) + if err != nil { + return nil, nil, err + } + + clientOptions, err := GetClientOptions(bslCfg, creds) + if err != nil { + return nil, nil, err + } + blobClientOptions := &azblob.ClientOptions{ + ClientOptions: clientOptions, + } + + // auth with storage account access key + accessKey := creds[CredentialKeyStorageAccountAccessKey] + if accessKey != "" { + log.Info("auth with the storage account access key") + cred, err := azblob.NewSharedKeyCredential(storageAccount, accessKey) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create storage account access key credential") + } + client, err := azblob.NewClientWithSharedKeyCredential(uri, cred, blobClientOptions) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create blob client with the storage account access key") + } + return client, cred, nil + } + + // auth with Azure AD + log.Info("auth with Azure AD") + cred, err := NewCredential(creds, clientOptions) + if err != nil { + return nil, nil, err + } + client, err := azblob.NewClient(uri, cred, blobClientOptions) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create blob client with the Azure AD credential") + } + return client, nil, nil +} + +// GetStorageAccountCredentials returns the credentials to interactive with storage account according to the config of BSL +// and credential file by the following order: +// 1. Return the storage account access key directly if it is provided +// 2. Return the content of the credential file directly if "userAAD" is set as true in BSL config +// 3. Call Azure API to exchange the storage account access key +func GetStorageAccountCredentials(bslCfg map[string]string, creds map[string]string) (map[string]string, error) { + // use storage account access key if specified + if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" { + accessKey := creds[name] + if accessKey == "" { + return nil, errors.Errorf("no storage account access key with key %s found in credential", name) + } + creds[CredentialKeyStorageAccountAccessKey] = accessKey + return creds, nil + } + + // use AAD + if bslCfg[BSLConfigUseAAD] != "" { + useAAD, err := strconv.ParseBool(bslCfg[BSLConfigUseAAD]) + if err != nil { + return nil, errors.Errorf("failed to parse bool from useAAD string: %s", bslCfg[BSLConfigUseAAD]) + } + + if useAAD { + return creds, nil + } + } + + // exchange the storage account access key + accessKey, err := exchangeStorageAccountAccessKey(bslCfg, creds) + if err != nil { + return nil, errors.WithMessage(err, "failed to get storage account access key") + } + creds[CredentialKeyStorageAccountAccessKey] = accessKey + return creds, nil +} + +// getStorageAccountURI returns the storage account URI by the following order: +// 1. Return the storage account URI directly if it is specified in BSL config +// 2. Try to call Azure API to get the storage account URI if possible(Background: https://github.com/vmware-tanzu/velero/issues/6163) +// 3. Fall back to return the default URI +func getStorageAccountURI(log logrus.FieldLogger, bslCfg map[string]string, creds map[string]string) (string, error) { + // if the URI is specified in the BSL, return it directly + uri := bslCfg[BSLConfigStorageAccountURI] + if uri != "" { + log.Infof("the storage account URI %q is specified in the BSL, use it directly", uri) + return uri, nil + } + + storageAccount := bslCfg[BSLConfigStorageAccount] + + cloudCfg, err := getCloudConfiguration(bslCfg, creds) + if err != nil { + return "", err + } + + // the default URI + uri = fmt.Sprintf("https://%s.%s", storageAccount, cloudCfg.Services[serviceNameBlob].Endpoint) + + // the storage account access key cannot be used to get the storage account properties, + // so fallback to the default URI + if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" && creds[name] != "" { + log.Infof("auth with the storage account access key, cannot retrieve the storage account properties, fallback to use the default URI %q", uri) + return uri, nil + } + + client, err := newStorageAccountManagemenClient(bslCfg, creds) + if err != nil { + log.Infof("failed to create the storage account management client: %v, fallback to use the default URI %q", err, uri) + return uri, nil + } + + resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) + // we cannot get the storage account properties without the resource group, so fallback to the default URI + if resourceGroup == "" { + log.Infof("resource group isn't set which is required to retrieve the storage account properties, fallback to use the default URI %q", uri) + return uri, nil + } + + properties, err := client.GetProperties(context.Background(), resourceGroup, storageAccount, nil) + // get error, fallback to the default URI + if err != nil { + log.Infof("failed to retrieve the storage account properties: %v, fallback to use the default URI %q", err, uri) + return uri, nil + } + + uri = *properties.Account.Properties.PrimaryEndpoints.Blob + log.Infof("use the storage account URI retrieved from the storage account properties %q", uri) + return uri, nil +} + +// try to exchange the storage account access key with the provided credentials +func exchangeStorageAccountAccessKey(bslCfg, creds map[string]string) (string, error) { + client, err := newStorageAccountManagemenClient(bslCfg, creds) + if err != nil { + return "", err + } + + resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) + if resourceGroup == "" { + return "", errors.New("resource group is required in BSL or credential to exchange the storage account access key") + } + storageAccount := bslCfg[BSLConfigStorageAccount] + if storageAccount == "" { + return "", errors.Errorf("%s is required in the BSL to exchange the storage account access key", BSLConfigStorageAccount) + } + + expand := "kerb" + resp, err := client.ListKeys(context.Background(), resourceGroup, storageAccount, &armstorage.AccountsClientListKeysOptions{ + Expand: &expand, + }) + if err != nil { + return "", errors.Wrap(err, "failed to list storage account access keys") + } + for _, key := range resp.Keys { + if key == nil || key.Permissions == nil { + continue + } + if strings.EqualFold(string(*key.Permissions), string(armstorage.KeyPermissionFull)) { + return *key.Value, nil + } + } + return "", errors.New("no storage key with Full permissions found") +} + +// new a management client for the storage account +func newStorageAccountManagemenClient(bslCfg map[string]string, creds map[string]string) (*armstorage.AccountsClient, error) { + clientOptions, err := GetClientOptions(bslCfg, creds) + if err != nil { + return nil, err + } + + cred, err := NewCredential(creds, clientOptions) + if err != nil { + return nil, errors.WithMessage(err, "failed to create Azure AD credential") + } + + subID := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigSubscriptionID, CredentialKeySubscriptionID) + if subID == "" { + return nil, errors.New("subscription ID is required in BSL or credential to create the storage account client") + } + + client, err := armstorage.NewAccountsClient(subID, cred, &arm.ClientOptions{ + ClientOptions: clientOptions, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to create the storage account client") + } + + return client, nil +} diff --git a/pkg/util/azure/storage_test.go b/pkg/util/azure/storage_test.go new file mode 100644 index 0000000000..e32b3e340f --- /dev/null +++ b/pkg/util/azure/storage_test.go @@ -0,0 +1,223 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewStorageClient(t *testing.T) { + log := logrus.New() + config := map[string]string{} + + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("AccessKey: YWNjZXNza2V5\nAZURE_TENANT_ID: tenantid\nAZURE_CLIENT_ID: clientid\nAZURE_CLIENT_SECRET: secret") + require.Nil(t, err) + + // storage account isn't specified + _, _, err = NewStorageClient(log, config) + require.NotNil(t, err) + + // auth with storage account access key + config = map[string]string{ + BSLConfigStorageAccount: "storage-account", + "credentialsFile": name, + BSLConfigStorageAccountAccessKeyName: "AccessKey", + } + client, credential, err := NewStorageClient(log, config) + require.Nil(t, err) + assert.NotNil(t, client) + assert.NotNil(t, credential) + + // auth with Azure AD + config = map[string]string{ + BSLConfigStorageAccount: "storage-account", + "credentialsFile": name, + "useAAD": "true", + } + client, credential, err = NewStorageClient(log, config) + require.Nil(t, err) + assert.NotNil(t, client) + assert.Nil(t, credential) +} + +func TestGetStorageAccountCredentials(t *testing.T) { + // use access secret but no secret specified + cfg := map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds := map[string]string{} + _, err := GetStorageAccountCredentials(cfg, creds) + require.NotNil(t, err) + + // use access secret + cfg = map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds = map[string]string{ + "KEY": "key", + } + m, err := GetStorageAccountCredentials(cfg, creds) + require.Nil(t, err) + assert.Equal(t, "key", m[CredentialKeyStorageAccountAccessKey]) + + // use AAD, but useAAD invalid + cfg = map[string]string{ + "useAAD": "invalid", + } + creds = map[string]string{} + _, err = GetStorageAccountCredentials(cfg, creds) + require.NotNil(t, err) + + // use AAD + cfg = map[string]string{ + "useAAD": "true", + } + creds = map[string]string{ + "KEY": "key", + } + m, err = GetStorageAccountCredentials(cfg, creds) + require.Nil(t, err) + assert.Equal(t, creds, m) +} + +func Test_getStorageAccountURI(t *testing.T) { + log := logrus.New() + + // URI specified + bslCfg := map[string]string{ + BSLConfigStorageAccountURI: "uri", + } + creds := map[string]string{} + uri, err := getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "uri", uri) + + // no URI specified, and auth with access key + bslCfg = map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds = map[string]string{ + "KEY": "value", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://.blob.core.windows.net", uri) + + // no URI specified, auth with AAD, resource group isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://.blob.core.windows.net", uri) + + // no URI specified, auth with AAD, resource group specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + BSLConfigStorageAccount: "account", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://account.blob.core.windows.net", uri) +} + +func Test_exchangeStorageAccountAccessKey(t *testing.T) { + // resource group isn't specified + bslCfg := map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds := map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err := exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) + + // storage account isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) + + // storage account specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + BSLConfigStorageAccount: "account", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) +} + +func Test_newStorageAccountManagemenClient(t *testing.T) { + // subscription ID isn't specified + bslCfg := map[string]string{} + creds := map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err := newStorageAccountManagemenClient(bslCfg, creds) + require.NotNil(t, err) + + // subscription ID isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = newStorageAccountManagemenClient(bslCfg, creds) + require.Nil(t, err) +} diff --git a/pkg/util/azure/testdata/certificate.pem b/pkg/util/azure/testdata/certificate.pem new file mode 100644 index 0000000000..4b66bfa021 --- /dev/null +++ b/pkg/util/azure/testdata/certificate.pem @@ -0,0 +1,49 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDL1hG+JYCfIPp3 +tlZ05J4pYIJ3Ckfs432bE3rYuWlR2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRY +OCI69s4+lP3DwR8uBCp9xyVkF8thXfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+Qf +oxAb6tx0kEc7V3ozBLWoIDJjfwJ3NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIr +Aa7pxHzo/Nd0U3e7z+DlBcJV7dY6TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bC +lG0u7unS7QOBMd6bOGkeL+Bc+n22slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpX +wj/Ek0F7AgMBAAECggEAblU3UWdXUcs2CCqIbcl52wfEVs8X05/n01MeAcWKvqYG +hvGcz7eLvhir5dQoXcF3VhybMrIe6C4WcBIiZSxGwxU+rwEP8YaLwX1UPfOrQM7s +sZTdFTLWfUslO3p7q300fdRA92iG9COMDZvkElh0cBvQksxs9sSr149l9vk+ymtC +uBhZtHG6Ki0BIMBNC9jGUqDuOatXl/dkK4tNjXrNJT7tVwzPaqnNALIWl6B+k9oQ +m1oNhSH2rvs9tw2ITXfIoIk9KdOMjQVUD43wKOaz0hNZhUsb1OFuls7UtRzaFcZH +rMd/M8DtA104QTTlHK+XS7r+nqdv7+ZyB+suTdM+oQKBgQDxCrJZU3hJ0eJ4VYhK +xGDfVGNpYxNkQ4CDB9fwRNbFr/Ck3kgzfE9QxTx1pJOolVmfuFmk9B86in4UNy91 +KdaqT79AU5RdOBXNN6tuMbLC0AVqe8sZq+1vWVVwbCstffxEMmyW1Ju/FLYPl2Zp +e5P96dBh5B3mXrQtpDJ0RkxxaQKBgQDYfE6tQQnQSs2ewD6ae8Mu6j8ueDlVoZ37 +vze1QdBasR26xu2H8XBt3u41zc524BwQsB1GE1tnC8ZylrqwVEayK4FesSQRCO6o +yK8QSdb06I5J4TaN+TppCDPLzstOh0Dmxp+iFUGoErb7AEOLAJ/VebhF9kBZObL/ +HYy4Es+bQwKBgHW/4vYuB3IQXNCp/+V+X1BZ+iJOaves3gekekF+b2itFSKFD8JO +9LQhVfKmTheptdmHhgtF0keXxhV8C+vxX1Ndl7EF41FSh5vzmQRAtPHkCvFEviex +TFD70/gSb1lO1UA/Xbqk69yBcprVPAtFejss0EYx2MVj+CLftmIEwW0ZAoGBAIMG +EVQ45eikLXjkn78+Iq7VZbIJX6IdNBH29I+GqsUJJ5Yw6fh6P3KwF3qG+mvmTfYn +sUAFXS+r58rYwVsRVsxlGmKmUc7hmhibhaEVH72QtvWuEiexbRG+viKfIVuA7t39 +3wXpWZiQ4yBdU4Pgt9wrVEU7ukyGaHiReOa7s90jAoGAJc0K7smn98YutQQ+g2ur +ybfnsl0YdsksaP2S2zvZUmNevKPrgnaIDDabOlhYYga+AK1G3FQ7/nefUgiIg1Nd +kr+T6Q4osS3xHB6Az9p/jaF4R2KaWN2nNVCn7ecsmPxDdM7k1vLxaT26vwO9OP5f +YU/5CeIzrfA5nQyPZkOXZBk= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUF2VIP4+AnEtb52KTCHbo4+fESfswDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0xOTEwMzAyMjQ2MjBaFw0yMjA4 +MTkyMjQ2MjBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDL1hG+JYCfIPp3tlZ05J4pYIJ3Ckfs432bE3rYuWlR +2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRYOCI69s4+lP3DwR8uBCp9xyVkF8th +XfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+QfoxAb6tx0kEc7V3ozBLWoIDJjfwJ3 +NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIrAa7pxHzo/Nd0U3e7z+DlBcJV7dY6 +TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bClG0u7unS7QOBMd6bOGkeL+Bc+n22 +slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpXwj/Ek0F7AgMBAAGjUzBRMB0GA1Ud +DgQWBBT6Mf9uXFB67bY2PeW3GCTKfkO7vDAfBgNVHSMEGDAWgBT6Mf9uXFB67bY2 +PeW3GCTKfkO7vDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCZ +1+kTISX85v9/ag7glavaPFUYsOSOOofl8gSzov7L01YL+srq7tXdvZmWrjQ/dnOY +h18rp9rb24vwIYxNioNG/M2cW1jBJwEGsDPOwdPV1VPcRmmUJW9kY130gRHBCd/N +qB7dIkcQnpNsxPIIWI+sRQp73U0ijhOByDnCNHLHon6vbfFTwkO1XggmV5BdZ3uQ +JNJyckILyNzlhmf6zhonMp4lVzkgxWsAm2vgdawd6dmBa+7Avb2QK9s+IdUSutFh +DgW2L12Obgh12Y4sf1iKQXA0RbZ2k+XQIz8EKZa7vJQY0ciYXSgB/BV3a96xX3cx +LIPL8Vam8Ytkopi3gsGA +-----END CERTIFICATE----- \ No newline at end of file diff --git a/pkg/util/azure/util.go b/pkg/util/azure/util.go new file mode 100644 index 0000000000..21c5ef0577 --- /dev/null +++ b/pkg/util/azure/util.go @@ -0,0 +1,109 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "os" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/joho/godotenv" + "github.com/pkg/errors" +) + +const ( + // the keys of Azure variables in credential + CredentialKeySubscriptionID = "AZURE_SUBSCRIPTION_ID" // #nosec + CredentialKeyResourceGroup = "AZURE_RESOURCE_GROUP" // #nosec + CredentialKeyCloudName = "AZURE_CLOUD_NAME" // #nosec + CredentialKeyStorageAccountAccessKey = "AZURE_STORAGE_KEY" // #nosec + CredentialKeyAdditionallyAllowedTenants = "AZURE_ADDITIONALLY_ALLOWED_TENANTS" // #nosec + CredentialKeyTenantID = "AZURE_TENANT_ID" // #nosec + CredentialKeyClientID = "AZURE_CLIENT_ID" // #nosec + CredentialKeyClientSecret = "AZURE_CLIENT_SECRET" // #nosec + CredentialKeyClientCertificatePath = "AZURE_CLIENT_CERTIFICATE_PATH" // #nosec + CredentialKeyClientCertificatePassword = "AZURE_CLIENT_CERTIFICATE_PASSWORD" // #nosec + CredentialKeySendCertChain = "AZURE_CLIENT_SEND_CERTIFICATE_CHAIN" // #nosec + CredentialKeyUsername = "AZURE_USERNAME" // #nosec + CredentialKeyPassword = "AZURE_PASSWORD" // #nosec + + credentialFile = "credentialsFile" +) + +// LoadCredentials gets the credential file from config and loads it into a map +func LoadCredentials(config map[string]string) (map[string]string, error) { + // the default credential file + credFile := os.Getenv("AZURE_CREDENTIALS_FILE") + + // use the credential file specified in the BSL spec if provided + if config != nil && config[credentialFile] != "" { + credFile = config[credentialFile] + } + + // put the credential file content into a map + creds, err := godotenv.Read(credFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to read credentials from file %s", credFile) + } + return creds, nil +} + +// GetClientOptions returns the client options based on the BSL/VSL config and credentials +func GetClientOptions(locationCfg, creds map[string]string) (policy.ClientOptions, error) { + cloudCfg, err := getCloudConfiguration(locationCfg, creds) + if err != nil { + return policy.ClientOptions{}, err + } + return policy.ClientOptions{ + Cloud: cloudCfg, + }, nil +} + +// getCloudConfiguration based on the BSL/VSL config and credentials +func getCloudConfiguration(locationCfg, creds map[string]string) (cloud.Configuration, error) { + name := creds[CredentialKeyCloudName] + activeDirectoryAuthorityURI := locationCfg[BSLConfigActiveDirectoryAuthorityURI] + + var cfg cloud.Configuration + switch strings.ToUpper(name) { + case "", "AZURECLOUD", "AZUREPUBLICCLOUD": + cfg = cloud.AzurePublic + case "AZURECHINACLOUD": + cfg = cloud.AzureChina + case "AZUREUSGOVERNMENT", "AZUREUSGOVERNMENTCLOUD": + cfg = cloud.AzureGovernment + default: + return cloud.Configuration{}, errors.New(fmt.Sprintf("unknown cloud: %s", name)) + } + if activeDirectoryAuthorityURI != "" { + cfg.ActiveDirectoryAuthorityHost = activeDirectoryAuthorityURI + } + return cfg, nil +} + +// GetFromLocationConfigOrCredential returns the value of the specified key from BSL/VSL config or credentials +// as some common configuration items can be set in BSL/VSL config or credential file(such as the subscription ID or resource group) +// Reading from BSL/VSL config takes first. +func GetFromLocationConfigOrCredential(cfg, creds map[string]string, cfgKey, credKey string) string { + value := cfg[cfgKey] + if value != "" { + return value + } + return creds[credKey] +} diff --git a/pkg/util/azure/util_test.go b/pkg/util/azure/util_test.go new file mode 100644 index 0000000000..28336501ac --- /dev/null +++ b/pkg/util/azure/util_test.go @@ -0,0 +1,199 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadCredentials(t *testing.T) { + // no credential file + _, err := LoadCredentials(nil) + require.NotNil(t, err) + + // specified credential file in the config + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("key: value") + require.Nil(t, err) + + config := map[string]string{ + "credentialsFile": name, + } + credentials, err := LoadCredentials(config) + require.Nil(t, err) + assert.Equal(t, "value", credentials["key"]) + + // use the default path defined via env variable + config = nil + os.Setenv("AZURE_CREDENTIALS_FILE", name) + credentials, err = LoadCredentials(config) + require.Nil(t, err) + assert.Equal(t, "value", credentials["key"]) +} + +func TestGetClientOptions(t *testing.T) { + // invalid cloud name + bslCfg := map[string]string{} + creds := map[string]string{ + CredentialKeyCloudName: "invalid", + } + _, err := GetClientOptions(bslCfg, creds) + require.NotNil(t, err) + + // valid + bslCfg = map[string]string{ + CredentialKeyCloudName: "", + } + creds = map[string]string{} + options, err := GetClientOptions(bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, options.Cloud, cloud.AzurePublic) +} + +func Test_getCloudConfiguration(t *testing.T) { + publicCloudWithADURI := cloud.AzurePublic + publicCloudWithADURI.ActiveDirectoryAuthorityHost = "https://example.com" + cases := []struct { + name string + bslCfg map[string]string + creds map[string]string + err bool + expected cloud.Configuration + }{ + { + name: "invalid cloud name", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "invalid", + }, + err: true, + }, + { + name: "null cloud name", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZURECLOUD", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREPUBLICCLOUD", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "azurecloud", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure China cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZURECHINACLOUD", + }, + err: false, + expected: cloud.AzureChina, + }, + { + name: "azure US government cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREUSGOVERNMENT", + }, + err: false, + expected: cloud.AzureGovernment, + }, + { + name: "azure US government cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREUSGOVERNMENTCLOUD", + }, + err: false, + expected: cloud.AzureGovernment, + }, + { + name: "AD authority URI provided", + bslCfg: map[string]string{ + BSLConfigActiveDirectoryAuthorityURI: "https://example.com", + }, + creds: map[string]string{ + CredentialKeyCloudName: "", + }, + err: false, + expected: publicCloudWithADURI, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg, err := getCloudConfiguration(c.bslCfg, c.creds) + require.Equal(t, c.err, err != nil) + if !c.err { + assert.Equal(t, c.expected, cfg) + } + }) + } +} + +func TestGetFromLocationConfigOrCredential(t *testing.T) { + // from cfg + cfg := map[string]string{ + "cfgkey": "value", + } + creds := map[string]string{} + cfgKey, credKey := "cfgkey", "credkey" + str := GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) + assert.Equal(t, "value", str) + + // from cred + cfg = map[string]string{} + creds = map[string]string{ + "credkey": "value", + } + str = GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) + assert.Equal(t, "value", str) +} From b598150cd1584a4e69184ffa4a0897e062b41232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Wed, 6 Sep 2023 09:13:22 +0800 Subject: [PATCH 2/2] Support setting CA cert for BSL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support setting CA cert for BSL Signed-off-by: Wenkai Yin(尹文开) --- pkg/repository/provider/unified_repo.go | 7 +-- pkg/repository/provider/unified_repo_test.go | 2 +- .../udmrepo/kopialib/backend/azure.go | 4 ++ .../backend/azure/azure_storage_wrapper.go | 2 +- pkg/repository/udmrepo/kopialib/backend/s3.go | 2 +- .../udmrepo/kopialib/backend/s3_test.go | 8 +-- pkg/repository/udmrepo/repo_options.go | 2 +- pkg/util/azure/util.go | 58 +++++++++++++++++-- pkg/util/azure/util_test.go | 14 ++++- 9 files changed, 82 insertions(+), 17 deletions(-) diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index f684cc4986..51b9ef2873 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -498,10 +498,6 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo result[udmrepo.StoreOptionS3Endpoint] = strings.Trim(s3URL, "/") result[udmrepo.StoreOptionS3DisableTLSVerify] = config["insecureSkipTLSVerify"] result[udmrepo.StoreOptionS3DisableTLS] = strconv.FormatBool(disableTLS) - - if backupLocation.Spec.ObjectStorage != nil && backupLocation.Spec.ObjectStorage.CACert != nil { - result[udmrepo.StoreOptionS3CustomCA] = base64.StdEncoding.EncodeToString(backupLocation.Spec.ObjectStorage.CACert) - } } else if backendType == repoconfig.AzureBackend { for k, v := range config { result[k] = v @@ -510,6 +506,9 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo result[udmrepo.StoreOptionOssBucket] = bucket result[udmrepo.StoreOptionPrefix] = prefix + if backupLocation.Spec.ObjectStorage != nil && backupLocation.Spec.ObjectStorage.CACert != nil { + result[udmrepo.StoreOptionCACert] = base64.StdEncoding.EncodeToString(backupLocation.Spec.ObjectStorage.CACert) + } result[udmrepo.StoreOptionOssRegion] = strings.Trim(region, "/") result[udmrepo.StoreOptionFsPath] = config["fspath"] diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 0910ef841c..ff6c31b4a7 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -384,7 +384,7 @@ func TestGetStorageVariables(t *testing.T) { "endpoint": "fake-url", "doNotUseTLS": "false", "skipTLSVerify": "false", - "customCA": base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04, 0x05}), + "caCert": base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04, 0x05}), }, }, { diff --git a/pkg/repository/udmrepo/kopialib/backend/azure.go b/pkg/repository/udmrepo/kopialib/backend/azure.go index ddf0e4d925..ea7659f93d 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure.go @@ -21,6 +21,7 @@ import ( "github.com/kopia/kopia/repo/blob" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure" ) @@ -29,6 +30,9 @@ type AzureBackend struct { } func (c *AzureBackend) Setup(ctx context.Context, flags map[string]string) error { + if flags[udmrepo.StoreOptionCACert] != "" { + flags["caCertEncoded"] = "true" + } c.option = azure.Option{ Config: flags, Limits: setupLimits(ctx, flags), diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go index 5e855611c2..046b0459a3 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go @@ -20,10 +20,10 @@ import ( "context" "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/blob/azure" "github.com/kopia/kopia/repo/blob/throttling" "github.com/sirupsen/logrus" - "github.com/kopia/kopia/repo/blob/azure" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" azureutil "github.com/vmware-tanzu/velero/pkg/util/azure" ) diff --git a/pkg/repository/udmrepo/kopialib/backend/s3.go b/pkg/repository/udmrepo/kopialib/backend/s3.go index 43a7fc821b..075be9af19 100644 --- a/pkg/repository/udmrepo/kopialib/backend/s3.go +++ b/pkg/repository/udmrepo/kopialib/backend/s3.go @@ -44,7 +44,7 @@ func (c *S3Backend) Setup(ctx context.Context, flags map[string]string) error { c.options.DoNotUseTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLS, flags) c.options.DoNotVerifyTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLSVerify, flags) c.options.SessionToken = optionalHaveString(udmrepo.StoreOptionS3Token, flags) - c.options.RootCA = optionalHaveBase64(ctx, udmrepo.StoreOptionS3CustomCA, flags) + c.options.RootCA = optionalHaveBase64(ctx, udmrepo.StoreOptionCACert, flags) c.options.Limits = setupLimits(ctx, flags) diff --git a/pkg/repository/udmrepo/kopialib/backend/s3_test.go b/pkg/repository/udmrepo/kopialib/backend/s3_test.go index 43a761688b..df96de36e9 100644 --- a/pkg/repository/udmrepo/kopialib/backend/s3_test.go +++ b/pkg/repository/udmrepo/kopialib/backend/s3_test.go @@ -95,8 +95,8 @@ func TestS3Setup(t *testing.T) { { name: "with wrong ca", flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionS3CustomCA: "fake-base-64", + udmrepo.StoreOptionOssBucket: "fake-bucket", + udmrepo.StoreOptionCACert: "fake-base-64", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", @@ -105,8 +105,8 @@ func TestS3Setup(t *testing.T) { { name: "with correct ca", flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionS3CustomCA: "ZmFrZS1jYQ==", + udmrepo.StoreOptionOssBucket: "fake-bucket", + udmrepo.StoreOptionCACert: "ZmFrZS1jYQ==", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", diff --git a/pkg/repository/udmrepo/repo_options.go b/pkg/repository/udmrepo/repo_options.go index 09acc49f72..af54e09477 100644 --- a/pkg/repository/udmrepo/repo_options.go +++ b/pkg/repository/udmrepo/repo_options.go @@ -42,7 +42,6 @@ const ( StoreOptionS3Endpoint = "endpoint" StoreOptionS3DisableTLS = "doNotUseTLS" StoreOptionS3DisableTLSVerify = "skipTLSVerify" - StoreOptionS3CustomCA = "customCA" StoreOptionFsPath = "fspath" @@ -50,6 +49,7 @@ const ( StoreOptionOssBucket = "bucket" StoreOptionOssRegion = "region" + StoreOptionCACert = "caCert" StoreOptionCredentialFile = "credFile" StoreOptionPrefix = "prefix" diff --git a/pkg/util/azure/util.go b/pkg/util/azure/util.go index 21c5ef0577..0b4cfa067a 100644 --- a/pkg/util/azure/util.go +++ b/pkg/util/azure/util.go @@ -17,9 +17,15 @@ limitations under the License. package azure import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" "fmt" + "net" + "net/http" "os" "strings" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" @@ -66,13 +72,57 @@ func LoadCredentials(config map[string]string) (map[string]string, error) { // GetClientOptions returns the client options based on the BSL/VSL config and credentials func GetClientOptions(locationCfg, creds map[string]string) (policy.ClientOptions, error) { + options := policy.ClientOptions{} + cloudCfg, err := getCloudConfiguration(locationCfg, creds) if err != nil { - return policy.ClientOptions{}, err + return options, err + } + options.Cloud = cloudCfg + + if locationCfg["caCert"] != "" { + certPool, _ := x509.SystemCertPool() + if certPool == nil { + certPool = x509.NewCertPool() + } + var caCert []byte + // As this function is used in both repository and plugin, the caCert isn't encoded + // when passing to the plugin while is encoded when works with repository, use one + // config item to distinguish these two cases + if locationCfg["caCertEncoded"] != "" { + caCert, err = base64.StdEncoding.DecodeString(locationCfg["caCert"]) + if err != nil { + return options, err + } + } else { + caCert = []byte(locationCfg["caCert"]) + } + + certPool.AppendCertsFromPEM(caCert) + + // https://github.com/Azure/azure-sdk-for-go/blob/sdk/azcore/v1.6.1/sdk/azcore/runtime/transport_default_http_client.go#L19 + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: certPool, + }, + } + options.Transport = &http.Client{ + Transport: transport, + } } - return policy.ClientOptions{ - Cloud: cloudCfg, - }, nil + + return options, nil } // getCloudConfiguration based on the BSL/VSL config and credentials diff --git a/pkg/util/azure/util_test.go b/pkg/util/azure/util_test.go index 28336501ac..e5a92f78cf 100644 --- a/pkg/util/azure/util_test.go +++ b/pkg/util/azure/util_test.go @@ -64,14 +64,26 @@ func TestGetClientOptions(t *testing.T) { _, err := GetClientOptions(bslCfg, creds) require.NotNil(t, err) - // valid + // specify caCert bslCfg = map[string]string{ CredentialKeyCloudName: "", + "caCert": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZTakNDQXpLZ0F3SUJBZ0lVWmcxbzRpWld2bVh5ekJrQ0J6SGdiODZGemtFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1VqRUxNQWtHQTFVRUJoTUNRMDR4RERBS0JnTlZCQWdNQTFCRlN6RVJNQThHQTFVRUJ3d0lRbVZwSUVwcApibWN4RHpBTkJnTlZCQW9NQmxaTmQyRnlaVEVSTUE4R0ExVUVBd3dJU0dGeVltOXlRMEV3SGhjTk1qTXdPVEEyCk1ESXpOakUyV2hjTk1qUXdPVEExTURJek5qRTJXakJYTVFzd0NRWURWUVFHRXdKRFRqRU1NQW9HQTFVRUNBd0QKVUVWTE1SRXdEd1lEVlFRSERBaENaV2tnU21sdVp6RVBNQTBHQTFVRUNnd0dWazEzWVhKbE1SWXdGQVlEVlFRRApEQTFJWVhKaWIzSk5ZVzVoWjJWeU1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBCnIrK1FHaHYvUnBDUTFIcncrMnYyQWNoaGhUVTVQL3hCd2RIWkZHWWJzMmxGbGtiL3oycEs2Y05ycFZmNUtmdjIKVUNpZEovMjFhZHc2SWNGZWxkSnFudU4rSlJaWXh5S0w0bzdRRGNVSk1sUTZJZk5kbEI0NUNwcGFBZVA4blVVTgo0YUwyV244b094L1pROTd2YmRXeERIR1FqZGR4N3p0Q09PaVZ0SEk4NS9Ka3kydTJnNmVhMklndmh1ZEVPZ3JtCjJzNU8zZlVtdHhSTEhwNnpDbURYZGZFUWg4ZFpndCs1d0RlazRWR2t4Zk81VG1tUHJ0LzBPTnVGYjJMWGVWV0sKeXkzVDFFTGNOSWhPSzQ1amhEejNnb2JhQzAwK0JTdzJMejVocXRxdUQ2RGxmME53TWtBQkt6d1dMZkROOXNrRApVazVYTmZNa2c0L3JhblRIWHlCNUNKSlk1akhORUtBQWlnM2NFSFNvejVjeGJqTE14VDhoMEk3MitEWldmUzFTCjU3Rm1SN2ZTNXk0QUcrU3Z1U3kvbktCUktJS2dQZ0t2Rjl6NktuL2ZPMTNsbk5LbHQwWU5mWFBFV2hZQytmMUoKTWpTOWc5eHBpYkZhM2Q0aHpOeWZhMWJHcUxtbkUwNVNpbWZNMVI1Z21Tamw1Z1FKQlltTHA3dWRLdjFDSUNRSwptQng2WG4vcnJEZHFiMndCRUNRSjBMbUo2SW5SaFZtT0s0WUdFeFRqZ1FRMldSWHYxMnhVK05GYWlZS3cxZkp0CkdaemFQeENxaG5JZXM5cGNPY0FjdmFHVngwSjlFYnRod0ltekdoTjBBREdCOVZaL1dFdHYvN0gwQ2xjOVlyT0gKNnRMb212b2pjQUZnN0xFbXZxeFFEOFFSTzlZZVdTTkgvV1REY1hVb3R5a0NBd0VBQWFNVE1CRXdEd1lEVlIwUgpCQWd3Qm9jRUNycFFxakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBZ0VBZnRVdmR1UFMvajhSaWx4ME5aelhSeEY3Ck9HZW9qU2JaQ1ZvamNPdnRNTVlMaFkxdDE2Y2laY1VWMGF4Z2FUWkkvak9WMGJPTEl3dmEvcVB2Z1RmSWZqL0gKVzhiMlNTRVdIUzZPSFFaR1BYNy9zVFVwQzB6QVcva2haN1FWR1BoWEcyK0V1NjFaNE95ejZ5dTRPdi9MYjlMUQpmMU9zTXhwandkbmhxazFKaERxUkpZbGIvZ05TRGZnVlN0YmhHVzVhb3paUlBBMUtqVXVaT3QzR2xQR09Wd3ZLCnpUcFFMdGVTUHNibTJMcUl2ZEg4dlgzK1kwcHIzdEdtdnExbWtIWUhYQTlBZWtYRkVsRHc4dGtZVHdLaEFqblUKZEFjWTFkTis5ODNiMDI0L0JQUXZKQlRTVjd4blEyUnlrUmMrVGxIL3B5RlM1cEtVbUF0aU9qTElxL2ZEMmJVagorTzlxT1hjK0c1b0xEaXlXWDRXSG9XdkZZdTdva1gwT1dGcHFETXFOcHlLUkRzQ1FENXViMEVQaVlVS0hnWEhiCnV3UXVtK0pRRUREdzRXL1kzZktnMW9TWW1XOHJndFNPZmtRQlQ0UnlaTUg2SzN6cFp5dVVsbmJUV0NWeEcyYVoKWVo0T2JpbUFGbVlveGRYdktWdFU0YUdlTjRoaXBvb2dzaXVXKzZYQ3Bqa2pWZlZuUEY4elZVNlZ3anRQVkkzKwpxdWxRNWJLS3lKYng3bk9NNXFob2svSmk2N1pyZDhob3ZwclhhRUdvakNDTVI3MllPWGVuMlB3bVlZZWNkQ2pyCnErSDdHNUV3ZXBoRWxrN3RWRWY4RVV4OEc1Mk9SVEtZMkF1dlRGVlliUC8yaTROS1FlMWdEWWZrWnNzUk1MajEKK0JCQVVJcnFVMnRuUHhwZW4vMD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", } creds = map[string]string{} options, err := GetClientOptions(bslCfg, creds) require.Nil(t, err) assert.Equal(t, options.Cloud, cloud.AzurePublic) + assert.NotNil(t, options.Transport) + + // doesn't specify caCert + bslCfg = map[string]string{ + CredentialKeyCloudName: "", + } + creds = map[string]string{} + options, err = GetClientOptions(bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, options.Cloud, cloud.AzurePublic) + assert.Nil(t, options.Transport) } func Test_getCloudConfiguration(t *testing.T) {