From 18678436538a83d5221130dfecb4d3fe84d3446f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Mon, 19 Jun 2023 08:37:58 +0800 Subject: [PATCH] Make Kopia support Azure AD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit introduces our own Azure storage provider by copying Kopia's Azure storage provider code and wrapping it rather than contributing to upstream based on the following considerations: 1. Velero needs the capability to interact with the repository concurrently while Kopia doesn't, this will increase the complexity of Kopia if we contribute to upstream 2. The configuration items provided by Velero and Kopia are conflict, e.g. Velero supports customizing storage account URI which is a full path while Kopia supports customizing storage account domain which is part of the URI. We need to consider the backward compatibility and upgrade case if we contribute to upstream which needs extra efforts 3. Contribute to upstream is a longer cycle when we need to introduce new changes. With this commit, we no longer depends on upstream for the Azure storage provider part and is easy for us to maintain Signed-off-by: Wenkai Yin(尹文开) --- changelogs/unreleased/6686-ywk253100 | 1 + go.mod | 91 +++--- go.sum | 196 +++++------ pkg/repository/config/azure.go | 219 +------------ pkg/repository/config/azure_test.go | 181 ++-------- pkg/repository/provider/unified_repo.go | 19 +- pkg/repository/provider/unified_repo_test.go | 140 ++------ .../udmrepo/kopialib/backend/azure.go | 31 +- .../kopialib/backend/azure/azure_options.go | 25 ++ .../kopialib/backend/azure/azure_storage.go | 308 ++++++++++++++++++ .../backend/azure/azure_storage_wrapper.go | 76 +++++ .../azure/azure_storage_wrapper_test.go | 79 +++++ .../kopialib/backend/azure/clock/now.go | 16 + .../kopialib/backend/azure/clock/now_prod.go | 11 + .../kopialib/backend/azure/iocopy/iocopy.go | 71 ++++ .../backend/azure/iocopy/iocopy_test.go | 101 ++++++ .../azure/timestampmeta/timestampmeta.go | 30 ++ .../azure/timestampmeta/timestampmeta_test.go | 23 ++ .../udmrepo/kopialib/backend/azure_test.go | 85 +---- pkg/repository/udmrepo/repo_options.go | 5 - pkg/util/azure/credential.go | 133 ++++++++ pkg/util/azure/credential_test.go | 96 ++++++ pkg/util/azure/storage.go | 276 ++++++++++++++++ pkg/util/azure/storage_test.go | 223 +++++++++++++ pkg/util/azure/testdata/certificate.pem | 49 +++ pkg/util/azure/util.go | 109 +++++++ pkg/util/azure/util_test.go | 199 +++++++++++ 27 files changed, 2074 insertions(+), 719 deletions(-) create mode 100644 changelogs/unreleased/6686-ywk253100 create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/azure_options.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/azure_storage.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper_test.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/clock/now.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/clock/now_prod.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy_test.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta.go create mode 100644 pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta_test.go create mode 100644 pkg/util/azure/credential.go create mode 100644 pkg/util/azure/credential_test.go create mode 100644 pkg/util/azure/storage.go create mode 100644 pkg/util/azure/storage_test.go create mode 100644 pkg/util/azure/testdata/certificate.pem create mode 100644 pkg/util/azure/util.go create mode 100644 pkg/util/azure/util_test.go diff --git a/changelogs/unreleased/6686-ywk253100 b/changelogs/unreleased/6686-ywk253100 new file mode 100644 index 00000000000..d2a79be6db8 --- /dev/null +++ b/changelogs/unreleased/6686-ywk253100 @@ -0,0 +1 @@ +Make Kopia support Azure AD \ No newline at end of file diff --git a/go.mod b/go.mod index 5f49f570bdb..aa12831bb63 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,18 @@ module github.com/vmware-tanzu/velero go 1.20 require ( - cloud.google.com/go/storage v1.30.1 + cloud.google.com/go/storage v1.31.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-sdk-for-go v67.2.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.3.0 - github.com/aws/aws-sdk-go v1.44.253 + github.com/aws/aws-sdk-go v1.44.256 github.com/bombsimon/logrusr/v3 v3.0.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/fatih/color v1.15.0 @@ -26,23 +30,23 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.20.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.0 + github.com/prometheus/client_golang v1.16.0 github.com/robfig/cron v1.1.0 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/vmware-tanzu/crash-diagnostics v0.3.7 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f - golang.org/x/mod v0.10.0 - golang.org/x/net v0.9.0 - golang.org/x/oauth2 v0.7.0 - golang.org/x/text v0.9.0 - google.golang.org/api v0.120.0 - google.golang.org/grpc v1.54.0 - google.golang.org/protobuf v1.30.0 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/mod v0.11.0 + golang.org/x/net v0.11.0 + golang.org/x/oauth2 v0.9.0 + golang.org/x/text v0.10.0 + google.golang.org/api v0.129.0 + google.golang.org/grpc v1.56.1 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.25.6 k8s.io/apiextensions-apiserver v0.24.2 @@ -58,13 +62,11 @@ require ( ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.0 // indirect + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.20.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect + cloud.google.com/go/iam v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect @@ -72,6 +74,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect @@ -81,7 +84,7 @@ require ( github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -93,9 +96,9 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.2 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.8.0 // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -103,18 +106,19 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/klauspost/reedsolomon v1.11.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/klauspost/reedsolomon v1.11.8 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.52 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/minio-go/v7 v7.0.59 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -125,30 +129,33 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/rs/xid v1.4.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/vladimirvivien/gexe v0.1.1 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect - go.opentelemetry.io/otel/trace v1.14.0 // indirect + go.opentelemetry.io/otel v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.8.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/term v0.9.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect diff --git a/go.sum b/go.sum index bdc7d0922a3..8d06738fe6b 100644 --- a/go.sum +++ b/go.sum @@ -19,24 +19,23 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= -cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.20.0 h1:cUOcywWuowO9It2i1KX1lIb0HH7gLv6nENKuZGnlcSo= +cloud.google.com/go/compute v1.20.0/go.mod h1:kn5BhC++qUWR/AM3Dn21myV7QbgqejW04cAOrtppaQI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -46,19 +45,25 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= +cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v67.2.0+incompatible h1:Uu/Ww6ernvPTrpq31kITVTIm/I5jlJ1wjtEH/bmSB2k= github.com/Azure/azure-sdk-for-go v67.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 h1:LcJtQjCXJUm1s7JpUHZvu+bpgURhCatxVNbGADXniX0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0/go.mod h1:+OgGVo0Httq7N5oayfvaLQ/Jq+2gJdqfp++Hyyl7Tws= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -104,6 +109,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= @@ -129,8 +136,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.44.253 h1:iqDd0okcH4ShfFexz2zzf4VmeDFf6NOMm07pHnEb8iY= -github.com/aws/aws-sdk-go v1.44.253/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -197,9 +204,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -261,8 +266,8 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= @@ -382,8 +387,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.2 h1:WVtYAYuYxKeYajAmThMRYWP6K3wXkcqbGHeUgeubUHY= -github.com/google/s2a-go v0.1.2/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -391,12 +396,12 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= -github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -482,14 +487,13 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= -github.com/klauspost/reedsolomon v1.11.7/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= +github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kopia/htmluibuild v0.0.0-20230326183719-f482ef17e2c9 h1:s5Wa89s8RlPjuwqd8K8kuf+T9Kz4+NsbKwR/pJ3PAT0= @@ -509,6 +513,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -532,8 +537,8 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -542,10 +547,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= -github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/minio-go/v7 v7.0.59 h1:lxIXwsTIcQkYoEG25rUJbzpmSB/oWeVDmxFo/uWUUsw= +github.com/minio/minio-go/v7 v7.0.59/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -569,7 +574,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -608,6 +612,8 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -623,40 +629,39 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -668,8 +673,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -713,8 +718,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tg123/go-htpasswd v1.2.1 h1:i4wfsX1KvvkyoMiHZzjS0VzbAPWfxzI8INcZAKtutoU= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -774,17 +779,19 @@ go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUz go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.starlark.net v0.0.0-20201006213952-227f4aabceb5 h1:ApvY/1gw+Yiqb/FKeks3KnVPWpkR3xzij82XPKLjJVw= @@ -792,8 +799,8 @@ go.starlark.net v0.0.0-20201006213952-227f4aabceb5/go.mod h1:f0znQkUKRrkk36XxWbG go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -822,8 +829,9 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -834,8 +842,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f h1:Al51T6tzvuh3oiwX11vex3QgJ2XTedFPGmbEVh8cdoc= -golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -863,8 +871,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -903,7 +911,6 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -923,8 +930,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -940,8 +947,8 @@ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= +golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -954,8 +961,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1023,6 +1030,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1032,20 +1040,21 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1055,9 +1064,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1161,8 +1171,8 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.120.0 h1:TTmhTei0mkR+kiBSW2UzZmAbkTaBfUUzfchyXnzG9Hs= -google.golang.org/api v0.120.0/go.mod h1:CrSvlNEFCFLae9ZUtL1z+61+rEBD7J/aCYwVYKZoWFU= +google.golang.org/api v0.129.0 h1:2XbdjjNfFPXQyufzQVwPf1RRnHH8Den2pfNE2jw7L8w= +google.golang.org/api v0.129.0/go.mod h1:dFjiXlanKwWE3612X97llhsoI36FAoIiRj3aTl5b/zE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1218,8 +1228,12 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1245,8 +1259,8 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1260,8 +1274,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/repository/config/azure.go b/pkg/repository/config/azure.go index 1c203330e13..6662d13c6d9 100644 --- a/pkg/repository/config/azure.go +++ b/pkg/repository/config/azure.go @@ -17,225 +17,34 @@ limitations under the License. package config import ( - "context" - "fmt" - "os" - "strings" - - storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" - "github.com/Azure/go-autorest/autorest/azure" - "github.com/Azure/go-autorest/autorest/azure/auth" - "github.com/joho/godotenv" "github.com/pkg/errors" -) - -const ( - subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID" - cloudNameEnvVar = "AZURE_CLOUD_NAME" - resourceGroupConfigKey = "resourceGroup" - - storageAccountConfigKey = "storageAccount" - storageAccountKeyEnvVarConfigKey = "storageAccountKeyEnvVar" - subscriptionIDConfigKey = "subscriptionId" - storageDomainConfigKey = "storageDomain" + "github.com/vmware-tanzu/velero/pkg/util/azure" ) -// getSubscriptionID gets the subscription ID from the 'config' map if it contains -// it, else from the AZURE_SUBSCRIPTION_ID environment variable. -func getSubscriptionID(config map[string]string) string { - if subscriptionID := config[subscriptionIDConfigKey]; subscriptionID != "" { - return subscriptionID - } - - return os.Getenv(subscriptionIDEnvVar) -} - -func getStorageAccountKey(config map[string]string) (string, error) { - credentialsFile := selectCredentialsFile(config) - - if err := loadCredentialsIntoEnv(credentialsFile); err != nil { - return "", err - } - - // Get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not - // exist, parseAzureEnvironment will return azure.PublicCloud. - env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar)) - if err != nil { - return "", errors.Wrap(err, "unable to parse azure cloud name environment variable") - } - - // Get storage key from secret using key config[storageAccountKeyEnvVarConfigKey]. If the config does not - // exist, continue obtaining it using API - if secretKeyEnvVar := config[storageAccountKeyEnvVarConfigKey]; secretKeyEnvVar != "" { - storageKey := os.Getenv(secretKeyEnvVar) - if storageKey == "" { - return "", errors.Errorf("no storage key secret with key %s found", secretKeyEnvVar) - } - - return storageKey, nil - } - - // get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable - subscriptionID := getSubscriptionID(config) - if subscriptionID == "" { - return "", errors.New("azure subscription ID not found in object store's config or in environment variable") - } - - // we need config["resourceGroup"], config["storageAccount"] - if err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil { - return "", errors.Wrap(err, "unable to get all required config values") - } - - // get authorizer from environment in the following order: - // 1. client credentials (AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET) - // 2. client certificate (AZURE_CERTIFICATE_PATH, AZURE_CERTIFICATE_PASSWORD) - // 3. username and password (AZURE_USERNAME, AZURE_PASSWORD) - // 4. MSI (managed service identity) - authorizer, err := auth.NewAuthorizerFromEnvironment() - if err != nil { - return "", errors.Wrap(err, "error getting authorizer from environment") - } - - // get storageAccountsClient - storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID) - storageAccountsClient.Authorizer = authorizer - - // get storage key - res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey], storagemgmt.Kerb) - if err != nil { - return "", errors.WithStack(err) - } - if res.Keys == nil || len(*res.Keys) == 0 { - return "", errors.New("No storage keys found") - } - - var storageKey string - for _, key := range *res.Keys { - // The ListKeys call returns e.g. "FULL" but the storagemgmt.Full constant in the SDK is defined as "Full". - if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) { - storageKey = *key.Value - break - } - } - - if storageKey == "" { - return "", errors.New("No storage key with Full permissions found") - } - - return storageKey, nil -} - -func mapLookup(data map[string]string) func(string) string { - return func(key string) string { - return data[key] - } -} - // GetAzureResticEnvVars gets the environment variables that restic // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based // on info in the provided object storage location config map. func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) { - storageAccountKey, err := getStorageAccountKey(config) - if err != nil { - return nil, err + storageAccount := config[azure.BSLConfigStorageAccount] + if storageAccount == "" { + return nil, errors.New("storageAccount is required in the BSL") } - if err := getRequiredValues(mapLookup(config), storageAccountConfigKey); err != nil { - return nil, errors.Wrap(err, "unable to get all required config values") - } - - return map[string]string{ - "AZURE_ACCOUNT_NAME": config[storageAccountConfigKey], - "AZURE_ACCOUNT_KEY": storageAccountKey, - }, nil -} - -// credentialsFileFromEnv retrieves the Azure credentials file from the environment. -func credentialsFileFromEnv() string { - return os.Getenv("AZURE_CREDENTIALS_FILE") -} - -// selectCredentialsFile selects the Azure credentials file to use, retrieving it -// from the given config or falling back to retrieving it from the environment. -func selectCredentialsFile(config map[string]string) string { - if credentialsFile, ok := config[CredentialsFileKey]; ok { - return credentialsFile - } - - return credentialsFileFromEnv() -} - -// loadCredentialsIntoEnv loads the variables in the given credentials -// file into the current environment. -func loadCredentialsIntoEnv(credentialsFile string) error { - if credentialsFile == "" { - return nil - } - - if err := godotenv.Overload(credentialsFile); err != nil { - return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) - } - - return nil -} - -// ParseAzureEnvironment returns an azure.Environment for the given cloud -// name, or azure.PublicCloud if cloudName is empty. -func parseAzureEnvironment(cloudName string) (*azure.Environment, error) { - if cloudName == "" { - return &azure.PublicCloud, nil - } - - env, err := azure.EnvironmentFromName(cloudName) - return &env, errors.WithStack(err) -} - -func getRequiredValues(getValue func(string) string, keys ...string) error { - missing := []string{} - results := map[string]string{} - - for _, key := range keys { - if val := getValue(key); val == "" { - missing = append(missing, key) - } else { - results[key] = val - } - } - - if len(missing) > 0 { - return errors.Errorf("the following keys do not have values: %s", strings.Join(missing, ", ")) - } - - return nil -} - -// GetAzureStorageDomain gets the Azure storage domain required by a Azure blob connection, -// if the provided credential file doesn't have the value, get it from system's environment variables -func GetAzureStorageDomain(config map[string]string) (string, error) { - credentialsFile := selectCredentialsFile(config) - - if err := loadCredentialsIntoEnv(credentialsFile); err != nil { - return "", err - } - - return getStorageDomainFromCloudName(os.Getenv(cloudNameEnvVar)) -} - -func GetAzureCredentials(config map[string]string) (string, string, error) { - storageAccountKey, err := getStorageAccountKey(config) + creds, err := azure.LoadCredentials(config) if err != nil { - return "", "", err + return nil, err } - return config[storageAccountConfigKey], storageAccountKey, nil -} - -func getStorageDomainFromCloudName(cloudName string) (string, error) { - env, err := parseAzureEnvironment(cloudName) + // restic doesn't support Azure AD, set it as false + config[azure.BSLConfigUseAAD] = "false" + credentials, err := azure.GetStorageAccountCredentials(config, creds) if err != nil { - return "", errors.Wrapf(err, "unable to parse azure env from cloud name %s", cloudName) + return nil, err } - return fmt.Sprintf("blob.%s", env.StorageEndpointSuffix), nil + return map[string]string{ + "AZURE_ACCOUNT_NAME": storageAccount, + "AZURE_ACCOUNT_KEY": credentials[azure.CredentialKeyStorageAccountAccessKey], + }, nil } diff --git a/pkg/repository/config/azure_test.go b/pkg/repository/config/azure_test.go index 87cd31bf154..c283197b3ae 100644 --- a/pkg/repository/config/azure_test.go +++ b/pkg/repository/config/azure_test.go @@ -1,12 +1,9 @@ /* Copyright the Velero contributors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,161 +15,37 @@ package config import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" -) - -// setAzureEnvironment sets the Azure credentials environment variable to the -// given value and returns a function to restore it to its previous value -func setAzureEnvironment(t *testing.T, value string) func() { - envVar := "AZURE_CREDENTIALS_FILE" - var cleanup func() - - if original, exists := os.LookupEnv(envVar); exists { - cleanup = func() { - require.NoError(t, os.Setenv(envVar, original), "failed to reset %s environment variable", envVar) - } - } else { - cleanup = func() { - require.NoError(t, os.Unsetenv(envVar), "failed to reset %s environment variable", envVar) - } - } - - require.NoError(t, os.Setenv(envVar, value), "failed to set %s environment variable", envVar) - - return cleanup -} -func TestSelectCredentialsFile(t *testing.T) { - testCases := []struct { - name string - config map[string]string - environment string - expected string - }{ - { - name: "when config is empty and environment variable is not set, no file is selected", - expected: "", - }, - { - name: "when config contains credentials file and environment variable is not set, file from config is selected", - config: map[string]string{ - "credentialsFile": "/tmp/credentials/path/to/secret", - }, - expected: "/tmp/credentials/path/to/secret", - }, - { - name: "when config is empty and environment variable is set, file from environment is selected", - environment: "/credentials/file/from/env", - expected: "/credentials/file/from/env", - }, - { - name: "when config contains credentials file and environment variable is set, file from config is selected", - config: map[string]string{ - "credentialsFile": "/tmp/credentials/path/to/secret", - }, - environment: "/credentials/file/from/env", - expected: "/tmp/credentials/path/to/secret", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - cleanup := setAzureEnvironment(t, tc.environment) - defer cleanup() - - selectedFile := selectCredentialsFile(tc.config) - require.Equal(t, tc.expected, selectedFile) - }) - } -} - -func TestGetStorageDomainFromCloudName(t *testing.T) { - testCases := []struct { - name string - cloudName string - expected string - expectedErr string - }{ - { - name: "get azure env fail", - cloudName: "fake-cloud", - expectedErr: "unable to parse azure env from cloud name fake-cloud: autorest/azure: There is no cloud environment matching the name \"FAKE-CLOUD\"", - }, - { - name: "cloud name is empty", - cloudName: "", - expected: "blob.core.windows.net", - }, - { - name: "azure public cloud", - cloudName: "AzurePublicCloud", - expected: "blob.core.windows.net", - }, - { - - name: "azure China cloud", - cloudName: "AzureChinaCloud", - expected: "blob.core.chinacloudapi.cn", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - domain, err := getStorageDomainFromCloudName(tc.cloudName) - - require.Equal(t, tc.expected, domain) - - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.expectedErr) - assert.Empty(t, domain) - } - }) - } -} - -func TestGetRequiredValues(t *testing.T) { - testCases := []struct { - name string - mp map[string]string - keys []string - err string - }{ - { - name: "with miss", - mp: map[string]string{ - "key1": "value1", - }, - keys: []string{"key1", "key2", "key3"}, - err: "the following keys do not have values: key2, key3", - }, - { - name: "without miss", - mp: map[string]string{ - "key1": "value1", - "key2": "value2", - "key3": "value3", - }, - keys: []string{"key1", "key2", "key3"}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := getRequiredValues(func(key string) string { - if tc.mp == nil { - return "" - } else { - return tc.mp[key] - } - }, tc.keys...) + "github.com/vmware-tanzu/velero/pkg/util/azure" +) - if err == nil { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.err) - } - }) - } +func TestGetAzureResticEnvVars(t *testing.T) { + config := map[string]string{} + + // no storage account specified + _, err := GetAzureResticEnvVars(config) + require.NotNil(t, err) + + // specify storage account access key + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("AccessKey: accesskey") + require.Nil(t, err) + + config[azure.BSLConfigStorageAccount] = "account01" + config[azure.BSLConfigStorageAccountAccessKeyName] = "AccessKey" + config["credentialsFile"] = name + envs, err := GetAzureResticEnvVars(config) + require.Nil(t, err) + + assert.Equal(t, "account01", envs["AZURE_ACCOUNT_NAME"]) + assert.Equal(t, "accesskey", envs["AZURE_ACCOUNT_KEY"]) } diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 8dc47018553..f684cc49861 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -47,11 +47,9 @@ type unifiedRepoProvider struct { // this func is assigned to a package-level variable so it can be // replaced when unit-testing -var getAzureCredentials = repoconfig.GetAzureCredentials var getS3Credentials = repoconfig.GetS3Credentials var getGCPCredentials = repoconfig.GetGCPCredentials var getS3BucketRegion = repoconfig.GetAWSBucketRegion -var getAzureStorageDomain = repoconfig.GetAzureStorageDomain type localFuncTable struct { getStorageVariables func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) @@ -190,6 +188,7 @@ func (urp *unifiedRepoProvider) PrepareRepo(ctx context.Context, param RepoParam log.Debug("Repo has already been initialized remotely") return nil } + log.Infof("failed to connect to the repo: %v, will try to create it", err) err = urp.repoService.Init(ctx, *repoOption, true) if err != nil { @@ -436,13 +435,8 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr result[udmrepo.StoreOptionS3Token] = credValue.SessionToken } case repoconfig.AzureBackend: - storageAccount, accountKey, err := getAzureCredentials(config) - if err != nil { - return map[string]string{}, errors.Wrap(err, "error get azure credentials") - } - result[udmrepo.StoreOptionAzureStorageAccount] = storageAccount - result[udmrepo.StoreOptionAzureKey] = accountKey - + // do nothing here, will retrieve the credential in Azure Storage + return nil, nil case repoconfig.GCPBackend: result[udmrepo.StoreOptionCredentialFile] = getGCPCredentials(config) } @@ -509,12 +503,9 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo result[udmrepo.StoreOptionS3CustomCA] = base64.StdEncoding.EncodeToString(backupLocation.Spec.ObjectStorage.CACert) } } else if backendType == repoconfig.AzureBackend { - domain, err := getAzureStorageDomain(config) - if err != nil { - return map[string]string{}, errors.Wrapf(err, "error to get azure storage domain") + for k, v := range config { + result[k] = v } - - result[udmrepo.StoreOptionAzureDomain] = domain } result[udmrepo.StoreOptionOssBucket] = bucket diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 08f33eb5758..0910ef841c7 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -39,16 +39,15 @@ import ( func TestGetStorageCredentials(t *testing.T) { testCases := []struct { - name string - backupLocation velerov1api.BackupStorageLocation - credFileStore *credmock.FileStore - credStoreError error - credStorePath string - getAzureCredentials func(map[string]string) (string, string, error) - getS3Credentials func(map[string]string) (*awscredentials.Value, error) - getGCPCredentials func(map[string]string) string - expected map[string]string - expectedErr string + name string + backupLocation velerov1api.BackupStorageLocation + credFileStore *credmock.FileStore + credStoreError error + credStorePath string + getS3Credentials func(map[string]string) (*awscredentials.Value, error) + getGCPCredentials func(map[string]string) string + expected map[string]string + expectedErr string }{ { name: "invalid credentials file store interface", @@ -160,43 +159,15 @@ func TestGetStorageCredentials(t *testing.T) { expected: map[string]string{}, }, { - name: "azure, Credential section exists in BSL", + name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", - }, + Provider: "velero.io/azure", Credential: &corev1api.SecretKeySelector{}, }, }, credFileStore: new(credmock.FileStore), - credStorePath: "credentials-from-credential-key", - getAzureCredentials: func(config map[string]string) (string, string, error) { - return "storage account from: " + config["credentialsFile"], "", nil - }, - - expected: map[string]string{ - "storageAccount": "storage account from: credentials-from-credential-key", - "storageKey": "", - }, - }, - { - name: "azure, get azure credentials fails", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", - }, - }, - }, - getAzureCredentials: func(config map[string]string) (string, string, error) { - return "", "", errors.New("fake error") - }, - credFileStore: new(credmock.FileStore), - expected: map[string]string{}, - expectedErr: "error get azure credentials: fake error", + expected: nil, }, { name: "gcp, Credential section not exists in BSL", @@ -220,7 +191,6 @@ func TestGetStorageCredentials(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - getAzureCredentials = tc.getAzureCredentials getS3Credentials = tc.getS3Credentials getGCPCredentials = tc.getGCPCredentials @@ -245,14 +215,14 @@ func TestGetStorageCredentials(t *testing.T) { func TestGetStorageVariables(t *testing.T) { testCases := []struct { - name string - backupLocation velerov1api.BackupStorageLocation - repoName string - repoBackend string - getS3BucketRegion func(string) (string, error) - getAzureStorageDomain func(map[string]string) (string, error) - expected map[string]string - expectedErr string + name string + backupLocation velerov1api.BackupStorageLocation + credFileStore *credmock.FileStore + repoName string + repoBackend string + getS3BucketRegion func(string) (string, error) + expected map[string]string + expectedErr string }{ { name: "invalid provider", @@ -418,7 +388,7 @@ func TestGetStorageVariables(t *testing.T) { }, }, { - name: "azure, getAzureStorageDomain fail", + name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/azure", @@ -436,68 +406,13 @@ func TestGetStorageVariables(t *testing.T) { }, }, }, - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "", errors.New("fake error") - }, - repoBackend: "fake-repo-type", - expected: map[string]string{}, - expectedErr: "error to get azure storage domain: fake error", - }, - { - name: "azure, ObjectStorage section exists in BSL", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "bucket": "fake-bucket-config", - "prefix": "fake-prefix-config", - "region": "fake-region", - "fspath": "", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "fake-bucket-object-store", - Prefix: "fake-prefix-object-store", - }, - }, - }, - }, - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "fake-domain", nil - }, - repoBackend: "fake-repo-type", - expected: map[string]string{ - "bucket": "fake-bucket-object-store", - "prefix": "fake-prefix-object-store/fake-repo-type/", - "region": "fake-region", - "fspath": "", - "storageDomain": "fake-domain", - }, - }, - { - name: "azure, ObjectStorage section not exists in BSL, repo name exists", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "bucket": "fake-bucket", - "prefix": "fake-prefix", - "region": "fake-region", - "fspath": "", - }, - }, - }, - repoName: "//fake-name//", - repoBackend: "fake-repo-type", - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "fake-domain", nil - }, + credFileStore: new(credmock.FileStore), + repoBackend: "fake-repo-type", expected: map[string]string{ - "bucket": "fake-bucket", - "prefix": "fake-prefix/fake-repo-type/fake-name/", - "region": "fake-region", - "fspath": "", - "storageDomain": "fake-domain", + "bucket": "fake-bucket-object-store", + "prefix": "fake-prefix-object-store/fake-repo-type/", + "region": "fake-region", + "fspath": "", }, }, { @@ -524,7 +439,6 @@ func TestGetStorageVariables(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { getS3BucketRegion = tc.getS3BucketRegion - getAzureStorageDomain = tc.getAzureStorageDomain actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName) diff --git a/pkg/repository/udmrepo/kopialib/backend/azure.go b/pkg/repository/udmrepo/kopialib/backend/azure.go index d5243820f40..ddf0e4d9252 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure.go @@ -20,41 +20,22 @@ import ( "context" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/blob/azure" - "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure" ) type AzureBackend struct { - options azure.Options + option azure.Option } func (c *AzureBackend) Setup(ctx context.Context, flags map[string]string) error { - var err error - c.options.Container, err = mustHaveString(udmrepo.StoreOptionOssBucket, flags) - if err != nil { - return err + c.option = azure.Option{ + Config: flags, + Limits: setupLimits(ctx, flags), } - - c.options.StorageAccount, err = mustHaveString(udmrepo.StoreOptionAzureStorageAccount, flags) - if err != nil { - return err - } - - c.options.StorageKey, err = mustHaveString(udmrepo.StoreOptionAzureKey, flags) - if err != nil { - return err - } - - c.options.Prefix = optionalHaveString(udmrepo.StoreOptionPrefix, flags) - c.options.SASToken = optionalHaveString(udmrepo.StoreOptionAzureToken, flags) - c.options.StorageDomain = optionalHaveString(udmrepo.StoreOptionAzureDomain, flags) - - c.options.Limits = setupLimits(ctx, flags) - return nil } func (c *AzureBackend) Connect(ctx context.Context, isCreate bool) (blob.Storage, error) { - return azure.New(ctx, &c.options, false) + return azure.NewStorage(ctx, &c.option, false) } diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/azure_options.go b/pkg/repository/udmrepo/kopialib/backend/azure/azure_options.go new file mode 100644 index 00000000000..c3b334542e7 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/azure_options.go @@ -0,0 +1,25 @@ +package azure + +import ( + "github.com/kopia/kopia/repo/blob/throttling" +) + +// Options defines options for Azure blob storage storage. +type Options struct { + // Container is the name of the azure storage container where data is stored. + Container string `json:"container"` + + // Prefix specifies additional string to prepend to all objects. + Prefix string `json:"prefix,omitempty"` + + // Azure Storage account name and key + StorageAccount string `json:"storageAccount"` + StorageKey string `json:"storageKey" kopia:"sensitive"` + + // Alternatively provide SAS Token + SASToken string `json:"sasToken" kopia:"sensitive"` + + StorageDomain string `json:"storageDomain,omitempty"` + + throttling.Limits +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage.go b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage.go new file mode 100644 index 00000000000..369c5c2a267 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage.go @@ -0,0 +1,308 @@ +// Package azure implements Azure Blob Storage. +package azure + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/blob/retrying" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure/clock" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure/iocopy" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta" +) + +const ( + azStorageType = "azureBlob" + + timeMapKey = "Kopiamtime" // this must be capital letter followed by lowercase, to comply with AZ tags naming convention. +) + +type azStorage struct { + Options + + service *azblob.Client + container string +} + +func (az *azStorage) GetCapacity(ctx context.Context) (blob.Capacity, error) { + return blob.Capacity{}, blob.ErrNotAVolume +} + +func (az *azStorage) GetBlob(ctx context.Context, b blob.ID, offset, length int64, output blob.OutputBuffer) error { + if offset < 0 { + return errors.Wrap(blob.ErrInvalidRange, "invalid offset") + } + + opt := &azblob.DownloadStreamOptions{} + + if length > 0 { + opt.Range.Offset = offset + opt.Range.Count = length + } + + if length == 0 { + l1 := int64(1) + opt.Range.Offset = offset + opt.Range.Count = l1 + } + + resp, err := az.service.DownloadStream(ctx, az.container, az.getObjectNameString(b), opt) + if err != nil { + return translateError(err) + } + + body := resp.Body + defer body.Close() //nolint:errcheck + + if length == 0 { + return nil + } + + if err := iocopy.JustCopy(output, body); err != nil { + return translateError(err) + } + + //nolint:wrapcheck + return blob.EnsureLengthExactly(output.Length(), length) +} + +func (az *azStorage) GetMetadata(ctx context.Context, b blob.ID) (blob.Metadata, error) { + bc := az.service.ServiceClient().NewContainerClient(az.container).NewBlobClient(az.getObjectNameString(b)) + + fi, err := bc.GetProperties(ctx, nil) + if err != nil { + return blob.Metadata{}, errors.Wrap(translateError(err), "Attributes") + } + + bm := blob.Metadata{ + BlobID: b, + Length: *fi.ContentLength, + Timestamp: *fi.LastModified, + } + + if fi.Metadata[timeMapKey] != nil { + if t, ok := timestampmeta.FromValue(*fi.Metadata[timeMapKey]); ok { + bm.Timestamp = t + } + } + + return bm, nil +} + +func translateError(err error) error { + if err == nil { + return nil + } + + var re *azcore.ResponseError + + if errors.As(err, &re) { + switch re.ErrorCode { + case string(bloberror.BlobNotFound): + return blob.ErrBlobNotFound + case string(bloberror.InvalidRange): + return blob.ErrInvalidRange + } + } + + return err +} + +func (az *azStorage) PutBlob(ctx context.Context, b blob.ID, data blob.Bytes, opts blob.PutOptions) error { + switch { + case opts.HasRetentionOptions(): + return errors.Wrap(blob.ErrUnsupportedPutBlobOption, "blob-retention") + case opts.DoNotRecreate: + return errors.Wrap(blob.ErrUnsupportedPutBlobOption, "do-not-recreate") + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + tsMetadata := timestampmeta.ToMap(opts.SetModTime, timeMapKey) + + metadata := make(map[string]*string, len(tsMetadata)) + + for k, v := range tsMetadata { + metadata[k] = to.Ptr(v) + } + + uso := &azblob.UploadStreamOptions{ + Metadata: metadata, + } + + resp, err := az.service.UploadStream(ctx, az.container, az.getObjectNameString(b), data.Reader(), uso) + if err != nil { + return translateError(err) + } + + if opts.GetModTime != nil { + *opts.GetModTime = *resp.LastModified + } + + return nil +} + +// DeleteBlob deletes azure blob from container with given ID. +func (az *azStorage) DeleteBlob(ctx context.Context, b blob.ID) error { + _, err := az.service.DeleteBlob(ctx, az.container, az.getObjectNameString(b), nil) + err = translateError(err) + + // don't return error if blob is already deleted + if errors.Is(err, blob.ErrBlobNotFound) { + return nil + } + + return err +} + +func (az *azStorage) getObjectNameString(b blob.ID) string { + return az.Prefix + string(b) +} + +// ListBlobs list azure blobs with given prefix. +func (az *azStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback func(blob.Metadata) error) error { + prefixStr := az.Prefix + string(prefix) + + pager := az.service.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefixStr, + Include: azblob.ListBlobsInclude{ + Metadata: true, + }, + }) + + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return translateError(err) + } + + for _, it := range page.Segment.BlobItems { + n := *it.Name + + bm := blob.Metadata{ + BlobID: blob.ID(n[len(az.Prefix):]), + Length: *it.Properties.ContentLength, + } + + // see if we have 'Kopiamtime' metadata, if so - trust it. + if t, ok := timestampmeta.FromValue(stringDefault(it.Metadata["kopiamtime"], "")); ok { + bm.Timestamp = t + } else { + bm.Timestamp = *it.Properties.LastModified + } + + if err := callback(bm); err != nil { + return err + } + } + } + + return nil +} + +func stringDefault(s *string, def string) string { + if s == nil { + return def + } + + return *s +} + +func (az *azStorage) ConnectionInfo() blob.ConnectionInfo { + return blob.ConnectionInfo{ + Type: azStorageType, + Config: &az.Options, + } +} + +func (az *azStorage) DisplayName() string { + return fmt.Sprintf("Azure: %v", az.Options.Container) +} + +func (az *azStorage) Close(ctx context.Context) error { + return nil +} + +func (az *azStorage) FlushCaches(ctx context.Context) error { + return nil +} + +// New creates new Azure Blob Storage-backed storage with specified options: +// +// - the 'Container', 'StorageAccount' and 'StorageKey' fields are required and all other parameters are optional. +func New(ctx context.Context, opt *Options, isCreate bool) (blob.Storage, error) { + _ = isCreate + + if opt.Container == "" { + return nil, errors.New("container name must be specified") + } + + var ( + service *azblob.Client + serviceErr error + ) + + storageDomain := opt.StorageDomain + if storageDomain == "" { + storageDomain = "blob.core.windows.net" + } + + storageHostname := fmt.Sprintf("%v.%v", opt.StorageAccount, storageDomain) + + switch { + case opt.SASToken != "": + service, serviceErr = azblob.NewClientWithNoCredential( + fmt.Sprintf("https://%s?%s", storageHostname, opt.SASToken), nil) + + case opt.StorageKey != "": + // create a credentials object. + cred, err := azblob.NewSharedKeyCredential(opt.StorageAccount, opt.StorageKey) + if err != nil { + return nil, errors.Wrap(err, "unable to initialize credentials") + } + + service, serviceErr = azblob.NewClientWithSharedKeyCredential( + fmt.Sprintf("https://%s/", storageHostname), cred, nil, + ) + + default: + return nil, errors.Errorf("either storage key or SAS token must be provided") + } + + if serviceErr != nil { + return nil, errors.Wrap(serviceErr, "opening azure service") + } + + raw := &azStorage{ + Options: *opt, + container: opt.Container, + service: service, + } + + az := retrying.NewWrapper(raw) + + // verify Azure connection is functional by listing blobs in a bucket, which will fail if the container + // does not exist. We list with a prefix that will not exist, to avoid iterating through any objects. + nonExistentPrefix := fmt.Sprintf("kopia-azure-storage-initializing-%v", clock.Now().UnixNano()) + if err := raw.ListBlobs(ctx, blob.ID(nonExistentPrefix), func(md blob.Metadata) error { + return nil + }); err != nil { + return nil, errors.Wrap(err, "unable to list from the bucket") + } + + return az, nil +} + +func init() { + blob.AddSupportedStorage(azStorageType, Options{}, New) +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go new file mode 100644 index 00000000000..6663b0d455c --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go @@ -0,0 +1,76 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/blob/throttling" + "github.com/sirupsen/logrus" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + azureutil "github.com/vmware-tanzu/velero/pkg/util/azure" +) + +const ( + storageType = "azure" +) + +func init() { + blob.AddSupportedStorage(storageType, Option{}, NewStorage) +} + +type Option struct { + Config map[string]string `json:"config" kopia:"sensitive"` + Limits throttling.Limits +} + +type Storage struct { + *azStorage + Option *Option +} + +func (s *Storage) ConnectionInfo() blob.ConnectionInfo { + return blob.ConnectionInfo{ + Type: storageType, + Config: s.Option, + } +} + +func NewStorage(ctx context.Context, option *Option, isCreate bool) (blob.Storage, error) { + cfg := option.Config + + client, _, err := azureutil.NewStorageClient(logrus.New(), cfg) + if err != nil { + return nil, err + } + + storage := &Storage{ + Option: option, + azStorage: &azStorage{ + Options: Options{ + Container: cfg[udmrepo.StoreOptionOssBucket], + Prefix: cfg[udmrepo.StoreOptionPrefix], + Limits: option.Limits, + }, + container: cfg[udmrepo.StoreOptionOssBucket], + service: client, + }, + } + return storage, nil +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper_test.go b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper_test.go new file mode 100644 index 00000000000..9ff0e674038 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper_test.go @@ -0,0 +1,79 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kopia/kopia/repo/blob/throttling" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + "github.com/vmware-tanzu/velero/pkg/util/azure" +) + +func TestConnectionInfo(t *testing.T) { + option := &Option{ + Config: map[string]string{"key": "value"}, + } + storage := &Storage{ + Option: option, + } + + info := storage.ConnectionInfo() + assert.Equal(t, storageType, info.Type) + assert.EqualValues(t, option, info.Config) +} + +func TestNewStorage(t *testing.T) { + limits := throttling.Limits{ + ReadsPerSecond: 100, + UploadBytesPerSecond: 200, + } + + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("AccessKey: YWNjZXNza2V5") + require.Nil(t, err) + + option := &Option{ + Config: map[string]string{ + azure.BSLConfigStorageAccount: "storage-account", + azure.BSLConfigStorageAccountAccessKeyName: "AccessKey", + udmrepo.StoreOptionOssBucket: "bucket", + udmrepo.StoreOptionPrefix: "prefix", + "credentialsFile": name, + }, + Limits: limits, + } + + storage, err := NewStorage(context.Background(), option, false) + require.Nil(t, err) + s, ok := storage.(*Storage) + require.True(t, ok) + assert.Equal(t, "bucket", s.container) + assert.Equal(t, "bucket", s.Options.Container) + assert.Equal(t, "prefix", s.Options.Prefix) + assert.Equal(t, limits, s.Options.Limits) +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/clock/now.go b/pkg/repository/udmrepo/kopialib/backend/azure/clock/now.go new file mode 100644 index 00000000000..59580bbd8e4 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/clock/now.go @@ -0,0 +1,16 @@ +// Package clock provides indirection for accessing current wall clock time. +// this is suitable for timestamps and long-term time operations, not short-term time measurements. +package clock + +import ( + "time" +) + +// discardMonotonicTime discards any monotonic time component of time, +// which behaves incorrectly when the computer goes to sleep and we want to measure durations +// between points in time. +// +// See https://go.googlesource.com/proposal/+/master/design/12914-monotonic.md +func discardMonotonicTime(t time.Time) time.Time { + return time.Unix(0, t.UnixNano()) +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/clock/now_prod.go b/pkg/repository/udmrepo/kopialib/backend/azure/clock/now_prod.go new file mode 100644 index 00000000000..01548c11b4b --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/clock/now_prod.go @@ -0,0 +1,11 @@ +//go:build !testing +// +build !testing + +package clock + +import "time" + +// Now returns current wall clock time. +func Now() time.Time { + return discardMonotonicTime(time.Now()) //nolint:forbidigo +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy.go b/pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy.go new file mode 100644 index 00000000000..30f34f8df7d --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy.go @@ -0,0 +1,71 @@ +// Package iocopy is a wrapper around io.Copy() that recycles shared buffers. +package iocopy + +import ( + "io" + "sync" +) + +// BufSize is the size (in bytes) of the shared copy buffers Kopia uses to copy data. +const BufSize = 65536 + +var ( + mu sync.Mutex //nolint:gochecknoglobals + + // +checklocks:mu + buffers [][]byte //nolint:gochecknoglobals +) + +// GetBuffer allocates new temporary buffer suitable for copying data. +func GetBuffer() []byte { + mu.Lock() + defer mu.Unlock() + + if len(buffers) == 0 { + return make([]byte, BufSize) + } + + var b []byte + + n := len(buffers) - 1 + b, buffers = buffers[n], buffers[0:n] + + return b +} + +// ReleaseBuffer releases the buffer back to the pool. +func ReleaseBuffer(b []byte) { + mu.Lock() + defer mu.Unlock() + + buffers = append(buffers, b) +} + +// Copy is equivalent to io.Copy(). +func Copy(dst io.Writer, src io.Reader) (int64, error) { + // If the reader has a WriteTo method, use it to do the copy. + // Avoids an allocation and a copy. + if wt, ok := src.(io.WriterTo); ok { + //nolint:wrapcheck + return wt.WriteTo(dst) + } + + // Similarly, if the writer has a ReadFrom method, use it to do the copy. + if rt, ok := dst.(io.ReaderFrom); ok { + //nolint:wrapcheck + return rt.ReadFrom(src) + } + + buf := GetBuffer() + defer ReleaseBuffer(buf) + + //nolint:wrapcheck + return io.CopyBuffer(dst, src, buf) +} + +// JustCopy is just like Copy() but does not return the number of bytes. +func JustCopy(dst io.Writer, src io.Reader) error { + _, err := Copy(dst, src) + + return err +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy_test.go b/pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy_test.go new file mode 100644 index 00000000000..324546ee2b2 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/iocopy/iocopy_test.go @@ -0,0 +1,101 @@ +package iocopy_test + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure/iocopy" +) + +const ( + testBuf = "Hello, World!" + lenTestBuf = len(testBuf) +) + +type errorWriter struct{} + +func (errorWriter) Write(p []byte) (n int, err error) { + return 0, errors.New("write error") //nolint:goerr113 +} + +func TestGetBuffer(t *testing.T) { + buf := iocopy.GetBuffer() + require.Len(t, buf, iocopy.BufSize) +} + +func TestReleaseBuffer(t *testing.T) { + buf := iocopy.GetBuffer() + iocopy.ReleaseBuffer(buf) + buf2 := iocopy.GetBuffer() + require.Equal(t, &buf[0], &buf2[0], "Buffer was not recycled after ReleaseBuffer") +} + +func TestCopy(t *testing.T) { + src := strings.NewReader(testBuf) + dst := &bytes.Buffer{} + + n, err := iocopy.Copy(dst, src) + require.NoError(t, err) + require.Equal(t, n, int64(lenTestBuf)) + require.Equal(t, dst.String(), testBuf) +} + +func TestJustCopy(t *testing.T) { + src := strings.NewReader(testBuf) + dst := &bytes.Buffer{} + + err := iocopy.JustCopy(dst, src) + require.NoError(t, err) + require.NoError(t, err) + require.Equal(t, dst.String(), testBuf) +} + +func TestCopyError(t *testing.T) { + src := strings.NewReader(testBuf) + dst := errorWriter{} + + _, err := iocopy.Copy(dst, src) + require.Error(t, err) +} + +func TestJustCopyError(t *testing.T) { + src := strings.NewReader(testBuf) + dst := errorWriter{} + + err := iocopy.JustCopy(dst, src) + require.Error(t, err) +} + +type customReader struct { + io.Reader +} + +func TestCustomReader(t *testing.T) { + src := customReader{strings.NewReader(testBuf)} + dst := &bytes.Buffer{} + + n, err := iocopy.Copy(dst, src) + require.NoError(t, err) + require.Equal(t, n, int64(lenTestBuf)) + require.Equal(t, dst.String(), testBuf) +} + +type customWriter struct { + io.Writer +} + +func TestCopyWithCustomReaderAndWriter(t *testing.T) { + src := customReader{strings.NewReader(testBuf)} + dst := &bytes.Buffer{} + customDst := customWriter{dst} + + n, err := iocopy.Copy(customDst, src) + require.NoError(t, err) + require.Equal(t, n, int64(lenTestBuf)) + require.Equal(t, dst.String(), testBuf) +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta.go b/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta.go new file mode 100644 index 00000000000..5361e20b376 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta.go @@ -0,0 +1,30 @@ +// Package timestampmeta provides utilities for preserving timestamps +// using per-blob key-value-pairs (metadata, tags, etc.) +package timestampmeta + +import ( + "strconv" + "time" +) + +// ToMap returns a map containing single entry representing the provided time or nil map +// if the time is zero. The key-value pair map should be stored alongside the blob. +func ToMap(t time.Time, mapKey string) map[string]string { + if t.IsZero() { + return nil + } + + return map[string]string{ + mapKey: strconv.FormatInt(t.UnixNano(), 10), + } +} + +// FromValue attempts to convert the provided value stored in metadata into time.Time. +func FromValue(v string) (t time.Time, ok bool) { + nanos, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return time.Time{}, false + } + + return time.Unix(0, nanos), true +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta_test.go b/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta_test.go new file mode 100644 index 00000000000..84b1c6af29c --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta/timestampmeta_test.go @@ -0,0 +1,23 @@ +package timestampmeta_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure/timestampmeta" +) + +var ( + timeValue = time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + storedValue = "1577934245000000000" +) + +func TestToMap(t *testing.T) { + require.Equal(t, map[string]string{ + "aaa": storedValue, + }, timestampmeta.ToMap(timeValue, "aaa")) + + require.Nil(t, timestampmeta.ToMap(time.Time{}, "aaa")) +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure_test.go b/pkg/repository/udmrepo/kopialib/backend/azure_test.go index bc4997fbe79..6814c635abf 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure_test.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure_test.go @@ -20,83 +20,28 @@ import ( "context" "testing" + "github.com/kopia/kopia/repo/blob/throttling" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" - - "github.com/kopia/kopia/repo/blob/azure" - "github.com/kopia/kopia/repo/blob/throttling" ) func TestAzureSetup(t *testing.T) { - testCases := []struct { - name string - flags map[string]string - expected azure.Options - expectedErr string - }{ - { - name: "must have bucket name", - flags: map[string]string{}, - expectedErr: "key " + udmrepo.StoreOptionOssBucket + " not found", - }, - { - name: "must have storage account", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - }, - expected: azure.Options{ - Container: "fake-bucket", - }, - expectedErr: "key " + udmrepo.StoreOptionAzureStorageAccount + " not found", - }, - { - name: "must have secret key", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionAzureStorageAccount: "fake-account", - }, - expected: azure.Options{ - Container: "fake-bucket", - StorageAccount: "fake-account", - }, - expectedErr: "key " + udmrepo.StoreOptionAzureKey + " not found", - }, - { - name: "with limits", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionAzureStorageAccount: "fake-account", - udmrepo.StoreOptionAzureKey: "fake-key", - udmrepo.ThrottleOptionReadOps: "100", - udmrepo.ThrottleOptionUploadBytes: "200", - }, - expected: azure.Options{ - Container: "fake-bucket", - StorageAccount: "fake-account", - StorageKey: "fake-key", - Limits: throttling.Limits{ - ReadsPerSecond: 100, - UploadBytesPerSecond: 200, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - azFlags := AzureBackend{} + backend := AzureBackend{} - err := azFlags.Setup(context.Background(), tc.flags) - - require.Equal(t, tc.expected, azFlags.options) - - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.expectedErr) - } - }) + flags := map[string]string{ + "key": "value", + udmrepo.ThrottleOptionReadOps: "100", + udmrepo.ThrottleOptionUploadBytes: "200", } + limits := throttling.Limits{ + ReadsPerSecond: 100, + UploadBytesPerSecond: 200, + } + + err := backend.Setup(context.Background(), flags) + require.Nil(t, err) + assert.Equal(t, flags, backend.option.Config) + assert.Equal(t, limits, backend.option.Limits) } diff --git a/pkg/repository/udmrepo/repo_options.go b/pkg/repository/udmrepo/repo_options.go index 9337018f239..09acc49f729 100644 --- a/pkg/repository/udmrepo/repo_options.go +++ b/pkg/repository/udmrepo/repo_options.go @@ -44,11 +44,6 @@ const ( StoreOptionS3DisableTLSVerify = "skipTLSVerify" StoreOptionS3CustomCA = "customCA" - StoreOptionAzureKey = "storageKey" - StoreOptionAzureDomain = "storageDomain" - StoreOptionAzureStorageAccount = "storageAccount" - StoreOptionAzureToken = "sasToken" - StoreOptionFsPath = "fspath" StoreOptionGcsReadonly = "readonly" diff --git a/pkg/util/azure/credential.go b/pkg/util/azure/credential.go new file mode 100644 index 00000000000..6d5e16ded3b --- /dev/null +++ b/pkg/util/azure/credential.go @@ -0,0 +1,133 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/pkg/errors" +) + +// NewCredential chains the config credential and workload identity credential +func NewCredential(creds map[string]string, options policy.ClientOptions) (azcore.TokenCredential, error) { + var ( + credential []azcore.TokenCredential + errMsgs []string + ) + + additionalTenants := []string{} + if tenants := creds[CredentialKeyAdditionallyAllowedTenants]; tenants != "" { + additionalTenants = strings.Split(tenants, ";") + } + + // config credential + cfgCred, err := newConfigCredential(creds, configCredentialOptions{ + ClientOptions: options, + AdditionallyAllowedTenants: additionalTenants, + }) + if err == nil { + credential = append(credential, cfgCred) + } else { + errMsgs = append(errMsgs, err.Error()) + } + + // workload identity credential + wic, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + AdditionallyAllowedTenants: additionalTenants, + ClientOptions: options, + }) + if err == nil { + credential = append(credential, wic) + } else { + errMsgs = append(errMsgs, err.Error()) + } + + if len(credential) == 0 { + return nil, errors.Errorf("failed to create Azure credential: %s", strings.Join(errMsgs, "\n\t")) + } + + return azidentity.NewChainedTokenCredential(credential, nil) +} + +type configCredentialOptions struct { + azcore.ClientOptions + AdditionallyAllowedTenants []string +} + +// newConfigCredential works same as the azidentity.EnvironmentCredential but reads the credentials from a map +// rather than environment variables. This is required for Velero to run B/R concurrently +// https://github.com/Azure/azure-sdk-for-go/blob/sdk/azidentity/v1.3.0/sdk/azidentity/environment_credential.go#L80 +func newConfigCredential(creds map[string]string, options configCredentialOptions) (azcore.TokenCredential, error) { + tenantID := creds[CredentialKeyTenantID] + if tenantID == "" { + return nil, errors.Errorf("%s is required", CredentialKeyTenantID) + } + clientID := creds[CredentialKeyClientID] + if clientID == "" { + return nil, errors.Errorf("%s is required", CredentialKeyClientID) + } + + // client secret + if clientSecret := creds[CredentialKeyClientSecret]; clientSecret != "" { + return azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, &azidentity.ClientSecretCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + }) + } + + // certificate + if certPath := creds[CredentialKeyClientCertificatePath]; certPath != "" { + certData, err := os.ReadFile(certPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read certificate file %s", certPath) + } + var password []byte + if v := creds[CredentialKeyClientCertificatePassword]; v != "" { + password = []byte(v) + } + certs, key, err := azidentity.ParseCertificates(certData, password) + if err != nil { + return nil, errors.Wrapf(err, "failed to load certificate from %s", certPath) + } + o := &azidentity.ClientCertificateCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + } + if v, ok := creds[CredentialKeySendCertChain]; ok { + o.SendCertificateChain = v == "1" || strings.ToLower(v) == "true" + } + return azidentity.NewClientCertificateCredential(tenantID, clientID, certs, key, o) + } + + // username/password + if username := creds[CredentialKeyUsername]; username != "" { + if password := creds[CredentialKeyPassword]; password != "" { + return azidentity.NewUsernamePasswordCredential(tenantID, clientID, username, password, + &azidentity.UsernamePasswordCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + }) + } + return nil, errors.Errorf("%s is required", CredentialKeyPassword) + } + + return nil, errors.New("incomplete credential configuration. Only AZURE_TENANT_ID and AZURE_CLIENT_ID are set") +} diff --git a/pkg/util/azure/credential_test.go b/pkg/util/azure/credential_test.go new file mode 100644 index 00000000000..4d6d7f0d33c --- /dev/null +++ b/pkg/util/azure/credential_test.go @@ -0,0 +1,96 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +func TestNewCredential(t *testing.T) { + options := policy.ClientOptions{} + + // no credentials + creds := map[string]string{} + _, err := NewCredential(creds, options) + require.NotNil(t, err) + + // config credential + creds = map[string]string{ + CredentialKeyTenantID: "tenantid", + CredentialKeyClientID: "clientid", + CredentialKeyClientSecret: "secret", + } + _, err = NewCredential(creds, options) + require.Nil(t, err) +} + +func Test_newConfigCredential(t *testing.T) { + options := configCredentialOptions{} + + // tenantID not specified + creds := map[string]string{} + _, err := newConfigCredential(creds, options) + require.NotNil(t, err) + + // clientID not specified + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + } + _, err = newConfigCredential(creds, options) + require.NotNil(t, err) + + // client secret + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyClientSecret: "secret", + } + credential, err := newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok := credential.(*azidentity.ClientSecretCredential) + require.True(t, ok) + + // client certificate + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyClientCertificatePath: "testdata/certificate.pem", + } + credential, err = newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok = credential.(*azidentity.ClientCertificateCredential) + require.True(t, ok) + + // username/password + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyUsername: "username", + CredentialKeyPassword: "password", + } + credential, err = newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok = credential.(*azidentity.UsernamePasswordCredential) + require.True(t, ok) +} diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go new file mode 100644 index 00000000000..49943a3f922 --- /dev/null +++ b/pkg/util/azure/storage.go @@ -0,0 +1,276 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + _ "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + // the keys of Azure BSL config: + // https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md + BSLConfigResourceGroup = "resourceGroup" + BSLConfigStorageAccount = "storageAccount" + BSLConfigStorageAccountAccessKeyName = "storageAccountKeyEnvVar" + BSLConfigSubscriptionID = "subscriptionId" + BSLConfigStorageAccountURI = "storageAccountURI" + BSLConfigUseAAD = "useAAD" + BSLConfigActiveDirectoryAuthorityURI = "activeDirectoryAuthorityURI" + + serviceNameBlob cloud.ServiceName = "blob" +) + +func init() { + cloud.AzureChina.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.chinacloudapi.cn", + } + cloud.AzureGovernment.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.usgovcloudapi.net", + } + cloud.AzurePublic.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.windows.net", + } +} + +// NewStorageClient creates a blob storage client(data plane) with the provided config which contains BSL config and the credential file name. +// The returned azblob.SharedKeyCredential is needed for Azure plugin to generate the SAS URL when auth with storage +// account access key +func NewStorageClient(log logrus.FieldLogger, config map[string]string) (*azblob.Client, *azblob.SharedKeyCredential, error) { + // rename to bslCfg for easy understanding + bslCfg := config + + // storage account is required + storageAccount := bslCfg[BSLConfigStorageAccount] + if storageAccount == "" { + return nil, nil, errors.Errorf("%s is required in BSL", BSLConfigStorageAccount) + } + + // read the credentials provided by users + creds, err := LoadCredentials(config) + if err != nil { + return nil, nil, err + } + // exchange the storage account access key if needed + creds, err = GetStorageAccountCredentials(bslCfg, creds) + if err != nil { + return nil, nil, err + } + + // get the storage account URI + uri, err := getStorageAccountURI(log, bslCfg, creds) + if err != nil { + return nil, nil, err + } + + clientOptions, err := GetClientOptions(bslCfg, creds) + if err != nil { + return nil, nil, err + } + blobClientOptions := &azblob.ClientOptions{ + ClientOptions: clientOptions, + } + + // auth with storage account access key + accessKey := creds[CredentialKeyStorageAccountAccessKey] + if accessKey != "" { + log.Info("auth with the storage account access key") + cred, err := azblob.NewSharedKeyCredential(storageAccount, accessKey) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create storage account access key credential") + } + client, err := azblob.NewClientWithSharedKeyCredential(uri, cred, blobClientOptions) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create blob client with the storage account access key") + } + return client, cred, nil + } + + // auth with Azure AD + log.Info("auth with Azure AD") + cred, err := NewCredential(creds, clientOptions) + if err != nil { + return nil, nil, err + } + client, err := azblob.NewClient(uri, cred, blobClientOptions) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create blob client with the Azure AD credential") + } + return client, nil, nil +} + +// GetStorageAccountCredentials returns the credentials to interactive with storage account according to the config of BSL +// and credential file by the following order: +// 1. Return the storage account access key directly if it is provided +// 2. Return the content of the credential file directly if "userAAD" is set as true in BSL config +// 3. Call Azure API to exchange the storage account access key +func GetStorageAccountCredentials(bslCfg map[string]string, creds map[string]string) (map[string]string, error) { + // use storage account access key if specified + if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" { + accessKey := creds[name] + if accessKey == "" { + return nil, errors.Errorf("no storage account access key with key %s found in credential", name) + } + creds[CredentialKeyStorageAccountAccessKey] = accessKey + return creds, nil + } + + // use AAD + if bslCfg[BSLConfigUseAAD] != "" { + useAAD, err := strconv.ParseBool(bslCfg[BSLConfigUseAAD]) + if err != nil { + return nil, errors.Errorf("failed to parse bool from useAAD string: %s", bslCfg[BSLConfigUseAAD]) + } + + if useAAD { + return creds, nil + } + } + + // exchange the storage account access key + accessKey, err := exchangeStorageAccountAccessKey(bslCfg, creds) + if err != nil { + return nil, errors.WithMessage(err, "failed to get storage account access key") + } + creds[CredentialKeyStorageAccountAccessKey] = accessKey + return creds, nil +} + +// getStorageAccountURI returns the storage account URI by the following order: +// 1. Return the storage account URI directly if it is specified in BSL config +// 2. Try to call Azure API to get the storage account URI if possible(Background: https://github.com/vmware-tanzu/velero/issues/6163) +// 3. Fall back to return the default URI +func getStorageAccountURI(log logrus.FieldLogger, bslCfg map[string]string, creds map[string]string) (string, error) { + // if the URI is specified in the BSL, return it directly + uri := bslCfg[BSLConfigStorageAccountURI] + if uri != "" { + log.Infof("the storage account URI %q is specified in the BSL, use it directly", uri) + return uri, nil + } + + storageAccount := bslCfg[BSLConfigStorageAccount] + + cloudCfg, err := getCloudConfiguration(bslCfg, creds) + if err != nil { + return "", err + } + + // the default URI + uri = fmt.Sprintf("https://%s.%s", storageAccount, cloudCfg.Services[serviceNameBlob].Endpoint) + + // the storage account access key cannot be used to get the storage account properties, + // so fallback to the default URI + if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" && creds[name] != "" { + log.Infof("auth with the storage account access key, cannot retrieve the storage account properties, fallback to use the default URI %q", uri) + return uri, nil + } + + client, err := newStorageAccountManagemenClient(bslCfg, creds) + if err != nil { + log.Infof("failed to create the storage account management client: %v, fallback to use the default URI %q", err, uri) + return uri, nil + } + + resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) + // we cannot get the storage account properties without the resource group, so fallback to the default URI + if resourceGroup == "" { + log.Infof("resource group isn't set which is required to retrieve the storage account properties, fallback to use the default URI %q", uri) + return uri, nil + } + + properties, err := client.GetProperties(context.Background(), resourceGroup, storageAccount, nil) + // get error, fallback to the default URI + if err != nil { + log.Infof("failed to retrieve the storage account properties: %v, fallback to use the default URI %q", err, uri) + return uri, nil + } + + uri = *properties.Account.Properties.PrimaryEndpoints.Blob + log.Infof("use the storage account URI retrieved from the storage account properties %q", uri) + return uri, nil +} + +// try to exchange the storage account access key with the provided credentials +func exchangeStorageAccountAccessKey(bslCfg, creds map[string]string) (string, error) { + client, err := newStorageAccountManagemenClient(bslCfg, creds) + if err != nil { + return "", err + } + + resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) + if resourceGroup == "" { + return "", errors.New("resource group is required in BSL or credential to exchange the storage account access key") + } + storageAccount := bslCfg[BSLConfigStorageAccount] + if storageAccount == "" { + return "", errors.Errorf("%s is required in the BSL to exchange the storage account access key", BSLConfigStorageAccount) + } + + expand := "kerb" + resp, err := client.ListKeys(context.Background(), resourceGroup, storageAccount, &armstorage.AccountsClientListKeysOptions{ + Expand: &expand, + }) + if err != nil { + return "", errors.Wrap(err, "failed to list storage account access keys") + } + for _, key := range resp.Keys { + if key == nil || key.Permissions == nil { + continue + } + if strings.EqualFold(string(*key.Permissions), string(armstorage.KeyPermissionFull)) { + return *key.Value, nil + } + } + return "", errors.New("no storage key with Full permissions found") +} + +// new a management client for the storage account +func newStorageAccountManagemenClient(bslCfg map[string]string, creds map[string]string) (*armstorage.AccountsClient, error) { + clientOptions, err := GetClientOptions(bslCfg, creds) + if err != nil { + return nil, err + } + + cred, err := NewCredential(creds, clientOptions) + if err != nil { + return nil, errors.WithMessage(err, "failed to create Azure AD credential") + } + + subID := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigSubscriptionID, CredentialKeySubscriptionID) + if subID == "" { + return nil, errors.New("subscription ID is required in BSL or credential to create the storage account client") + } + + client, err := armstorage.NewAccountsClient(subID, cred, &arm.ClientOptions{ + ClientOptions: clientOptions, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to create the storage account client") + } + + return client, nil +} diff --git a/pkg/util/azure/storage_test.go b/pkg/util/azure/storage_test.go new file mode 100644 index 00000000000..e32b3e340fa --- /dev/null +++ b/pkg/util/azure/storage_test.go @@ -0,0 +1,223 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewStorageClient(t *testing.T) { + log := logrus.New() + config := map[string]string{} + + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("AccessKey: YWNjZXNza2V5\nAZURE_TENANT_ID: tenantid\nAZURE_CLIENT_ID: clientid\nAZURE_CLIENT_SECRET: secret") + require.Nil(t, err) + + // storage account isn't specified + _, _, err = NewStorageClient(log, config) + require.NotNil(t, err) + + // auth with storage account access key + config = map[string]string{ + BSLConfigStorageAccount: "storage-account", + "credentialsFile": name, + BSLConfigStorageAccountAccessKeyName: "AccessKey", + } + client, credential, err := NewStorageClient(log, config) + require.Nil(t, err) + assert.NotNil(t, client) + assert.NotNil(t, credential) + + // auth with Azure AD + config = map[string]string{ + BSLConfigStorageAccount: "storage-account", + "credentialsFile": name, + "useAAD": "true", + } + client, credential, err = NewStorageClient(log, config) + require.Nil(t, err) + assert.NotNil(t, client) + assert.Nil(t, credential) +} + +func TestGetStorageAccountCredentials(t *testing.T) { + // use access secret but no secret specified + cfg := map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds := map[string]string{} + _, err := GetStorageAccountCredentials(cfg, creds) + require.NotNil(t, err) + + // use access secret + cfg = map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds = map[string]string{ + "KEY": "key", + } + m, err := GetStorageAccountCredentials(cfg, creds) + require.Nil(t, err) + assert.Equal(t, "key", m[CredentialKeyStorageAccountAccessKey]) + + // use AAD, but useAAD invalid + cfg = map[string]string{ + "useAAD": "invalid", + } + creds = map[string]string{} + _, err = GetStorageAccountCredentials(cfg, creds) + require.NotNil(t, err) + + // use AAD + cfg = map[string]string{ + "useAAD": "true", + } + creds = map[string]string{ + "KEY": "key", + } + m, err = GetStorageAccountCredentials(cfg, creds) + require.Nil(t, err) + assert.Equal(t, creds, m) +} + +func Test_getStorageAccountURI(t *testing.T) { + log := logrus.New() + + // URI specified + bslCfg := map[string]string{ + BSLConfigStorageAccountURI: "uri", + } + creds := map[string]string{} + uri, err := getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "uri", uri) + + // no URI specified, and auth with access key + bslCfg = map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds = map[string]string{ + "KEY": "value", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://.blob.core.windows.net", uri) + + // no URI specified, auth with AAD, resource group isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://.blob.core.windows.net", uri) + + // no URI specified, auth with AAD, resource group specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + BSLConfigStorageAccount: "account", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://account.blob.core.windows.net", uri) +} + +func Test_exchangeStorageAccountAccessKey(t *testing.T) { + // resource group isn't specified + bslCfg := map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds := map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err := exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) + + // storage account isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) + + // storage account specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + BSLConfigStorageAccount: "account", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) +} + +func Test_newStorageAccountManagemenClient(t *testing.T) { + // subscription ID isn't specified + bslCfg := map[string]string{} + creds := map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err := newStorageAccountManagemenClient(bslCfg, creds) + require.NotNil(t, err) + + // subscription ID isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = newStorageAccountManagemenClient(bslCfg, creds) + require.Nil(t, err) +} diff --git a/pkg/util/azure/testdata/certificate.pem b/pkg/util/azure/testdata/certificate.pem new file mode 100644 index 00000000000..4b66bfa021a --- /dev/null +++ b/pkg/util/azure/testdata/certificate.pem @@ -0,0 +1,49 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDL1hG+JYCfIPp3 +tlZ05J4pYIJ3Ckfs432bE3rYuWlR2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRY +OCI69s4+lP3DwR8uBCp9xyVkF8thXfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+Qf +oxAb6tx0kEc7V3ozBLWoIDJjfwJ3NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIr +Aa7pxHzo/Nd0U3e7z+DlBcJV7dY6TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bC +lG0u7unS7QOBMd6bOGkeL+Bc+n22slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpX +wj/Ek0F7AgMBAAECggEAblU3UWdXUcs2CCqIbcl52wfEVs8X05/n01MeAcWKvqYG +hvGcz7eLvhir5dQoXcF3VhybMrIe6C4WcBIiZSxGwxU+rwEP8YaLwX1UPfOrQM7s +sZTdFTLWfUslO3p7q300fdRA92iG9COMDZvkElh0cBvQksxs9sSr149l9vk+ymtC +uBhZtHG6Ki0BIMBNC9jGUqDuOatXl/dkK4tNjXrNJT7tVwzPaqnNALIWl6B+k9oQ +m1oNhSH2rvs9tw2ITXfIoIk9KdOMjQVUD43wKOaz0hNZhUsb1OFuls7UtRzaFcZH +rMd/M8DtA104QTTlHK+XS7r+nqdv7+ZyB+suTdM+oQKBgQDxCrJZU3hJ0eJ4VYhK +xGDfVGNpYxNkQ4CDB9fwRNbFr/Ck3kgzfE9QxTx1pJOolVmfuFmk9B86in4UNy91 +KdaqT79AU5RdOBXNN6tuMbLC0AVqe8sZq+1vWVVwbCstffxEMmyW1Ju/FLYPl2Zp +e5P96dBh5B3mXrQtpDJ0RkxxaQKBgQDYfE6tQQnQSs2ewD6ae8Mu6j8ueDlVoZ37 +vze1QdBasR26xu2H8XBt3u41zc524BwQsB1GE1tnC8ZylrqwVEayK4FesSQRCO6o +yK8QSdb06I5J4TaN+TppCDPLzstOh0Dmxp+iFUGoErb7AEOLAJ/VebhF9kBZObL/ +HYy4Es+bQwKBgHW/4vYuB3IQXNCp/+V+X1BZ+iJOaves3gekekF+b2itFSKFD8JO +9LQhVfKmTheptdmHhgtF0keXxhV8C+vxX1Ndl7EF41FSh5vzmQRAtPHkCvFEviex +TFD70/gSb1lO1UA/Xbqk69yBcprVPAtFejss0EYx2MVj+CLftmIEwW0ZAoGBAIMG +EVQ45eikLXjkn78+Iq7VZbIJX6IdNBH29I+GqsUJJ5Yw6fh6P3KwF3qG+mvmTfYn +sUAFXS+r58rYwVsRVsxlGmKmUc7hmhibhaEVH72QtvWuEiexbRG+viKfIVuA7t39 +3wXpWZiQ4yBdU4Pgt9wrVEU7ukyGaHiReOa7s90jAoGAJc0K7smn98YutQQ+g2ur +ybfnsl0YdsksaP2S2zvZUmNevKPrgnaIDDabOlhYYga+AK1G3FQ7/nefUgiIg1Nd +kr+T6Q4osS3xHB6Az9p/jaF4R2KaWN2nNVCn7ecsmPxDdM7k1vLxaT26vwO9OP5f +YU/5CeIzrfA5nQyPZkOXZBk= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUF2VIP4+AnEtb52KTCHbo4+fESfswDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0xOTEwMzAyMjQ2MjBaFw0yMjA4 +MTkyMjQ2MjBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDL1hG+JYCfIPp3tlZ05J4pYIJ3Ckfs432bE3rYuWlR +2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRYOCI69s4+lP3DwR8uBCp9xyVkF8th +XfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+QfoxAb6tx0kEc7V3ozBLWoIDJjfwJ3 +NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIrAa7pxHzo/Nd0U3e7z+DlBcJV7dY6 +TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bClG0u7unS7QOBMd6bOGkeL+Bc+n22 +slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpXwj/Ek0F7AgMBAAGjUzBRMB0GA1Ud +DgQWBBT6Mf9uXFB67bY2PeW3GCTKfkO7vDAfBgNVHSMEGDAWgBT6Mf9uXFB67bY2 +PeW3GCTKfkO7vDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCZ +1+kTISX85v9/ag7glavaPFUYsOSOOofl8gSzov7L01YL+srq7tXdvZmWrjQ/dnOY +h18rp9rb24vwIYxNioNG/M2cW1jBJwEGsDPOwdPV1VPcRmmUJW9kY130gRHBCd/N +qB7dIkcQnpNsxPIIWI+sRQp73U0ijhOByDnCNHLHon6vbfFTwkO1XggmV5BdZ3uQ +JNJyckILyNzlhmf6zhonMp4lVzkgxWsAm2vgdawd6dmBa+7Avb2QK9s+IdUSutFh +DgW2L12Obgh12Y4sf1iKQXA0RbZ2k+XQIz8EKZa7vJQY0ciYXSgB/BV3a96xX3cx +LIPL8Vam8Ytkopi3gsGA +-----END CERTIFICATE----- \ No newline at end of file diff --git a/pkg/util/azure/util.go b/pkg/util/azure/util.go new file mode 100644 index 00000000000..21c5ef05772 --- /dev/null +++ b/pkg/util/azure/util.go @@ -0,0 +1,109 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "os" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/joho/godotenv" + "github.com/pkg/errors" +) + +const ( + // the keys of Azure variables in credential + CredentialKeySubscriptionID = "AZURE_SUBSCRIPTION_ID" // #nosec + CredentialKeyResourceGroup = "AZURE_RESOURCE_GROUP" // #nosec + CredentialKeyCloudName = "AZURE_CLOUD_NAME" // #nosec + CredentialKeyStorageAccountAccessKey = "AZURE_STORAGE_KEY" // #nosec + CredentialKeyAdditionallyAllowedTenants = "AZURE_ADDITIONALLY_ALLOWED_TENANTS" // #nosec + CredentialKeyTenantID = "AZURE_TENANT_ID" // #nosec + CredentialKeyClientID = "AZURE_CLIENT_ID" // #nosec + CredentialKeyClientSecret = "AZURE_CLIENT_SECRET" // #nosec + CredentialKeyClientCertificatePath = "AZURE_CLIENT_CERTIFICATE_PATH" // #nosec + CredentialKeyClientCertificatePassword = "AZURE_CLIENT_CERTIFICATE_PASSWORD" // #nosec + CredentialKeySendCertChain = "AZURE_CLIENT_SEND_CERTIFICATE_CHAIN" // #nosec + CredentialKeyUsername = "AZURE_USERNAME" // #nosec + CredentialKeyPassword = "AZURE_PASSWORD" // #nosec + + credentialFile = "credentialsFile" +) + +// LoadCredentials gets the credential file from config and loads it into a map +func LoadCredentials(config map[string]string) (map[string]string, error) { + // the default credential file + credFile := os.Getenv("AZURE_CREDENTIALS_FILE") + + // use the credential file specified in the BSL spec if provided + if config != nil && config[credentialFile] != "" { + credFile = config[credentialFile] + } + + // put the credential file content into a map + creds, err := godotenv.Read(credFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to read credentials from file %s", credFile) + } + return creds, nil +} + +// GetClientOptions returns the client options based on the BSL/VSL config and credentials +func GetClientOptions(locationCfg, creds map[string]string) (policy.ClientOptions, error) { + cloudCfg, err := getCloudConfiguration(locationCfg, creds) + if err != nil { + return policy.ClientOptions{}, err + } + return policy.ClientOptions{ + Cloud: cloudCfg, + }, nil +} + +// getCloudConfiguration based on the BSL/VSL config and credentials +func getCloudConfiguration(locationCfg, creds map[string]string) (cloud.Configuration, error) { + name := creds[CredentialKeyCloudName] + activeDirectoryAuthorityURI := locationCfg[BSLConfigActiveDirectoryAuthorityURI] + + var cfg cloud.Configuration + switch strings.ToUpper(name) { + case "", "AZURECLOUD", "AZUREPUBLICCLOUD": + cfg = cloud.AzurePublic + case "AZURECHINACLOUD": + cfg = cloud.AzureChina + case "AZUREUSGOVERNMENT", "AZUREUSGOVERNMENTCLOUD": + cfg = cloud.AzureGovernment + default: + return cloud.Configuration{}, errors.New(fmt.Sprintf("unknown cloud: %s", name)) + } + if activeDirectoryAuthorityURI != "" { + cfg.ActiveDirectoryAuthorityHost = activeDirectoryAuthorityURI + } + return cfg, nil +} + +// GetFromLocationConfigOrCredential returns the value of the specified key from BSL/VSL config or credentials +// as some common configuration items can be set in BSL/VSL config or credential file(such as the subscription ID or resource group) +// Reading from BSL/VSL config takes first. +func GetFromLocationConfigOrCredential(cfg, creds map[string]string, cfgKey, credKey string) string { + value := cfg[cfgKey] + if value != "" { + return value + } + return creds[credKey] +} diff --git a/pkg/util/azure/util_test.go b/pkg/util/azure/util_test.go new file mode 100644 index 00000000000..28336501acf --- /dev/null +++ b/pkg/util/azure/util_test.go @@ -0,0 +1,199 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadCredentials(t *testing.T) { + // no credential file + _, err := LoadCredentials(nil) + require.NotNil(t, err) + + // specified credential file in the config + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("key: value") + require.Nil(t, err) + + config := map[string]string{ + "credentialsFile": name, + } + credentials, err := LoadCredentials(config) + require.Nil(t, err) + assert.Equal(t, "value", credentials["key"]) + + // use the default path defined via env variable + config = nil + os.Setenv("AZURE_CREDENTIALS_FILE", name) + credentials, err = LoadCredentials(config) + require.Nil(t, err) + assert.Equal(t, "value", credentials["key"]) +} + +func TestGetClientOptions(t *testing.T) { + // invalid cloud name + bslCfg := map[string]string{} + creds := map[string]string{ + CredentialKeyCloudName: "invalid", + } + _, err := GetClientOptions(bslCfg, creds) + require.NotNil(t, err) + + // valid + bslCfg = map[string]string{ + CredentialKeyCloudName: "", + } + creds = map[string]string{} + options, err := GetClientOptions(bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, options.Cloud, cloud.AzurePublic) +} + +func Test_getCloudConfiguration(t *testing.T) { + publicCloudWithADURI := cloud.AzurePublic + publicCloudWithADURI.ActiveDirectoryAuthorityHost = "https://example.com" + cases := []struct { + name string + bslCfg map[string]string + creds map[string]string + err bool + expected cloud.Configuration + }{ + { + name: "invalid cloud name", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "invalid", + }, + err: true, + }, + { + name: "null cloud name", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZURECLOUD", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREPUBLICCLOUD", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "azurecloud", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure China cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZURECHINACLOUD", + }, + err: false, + expected: cloud.AzureChina, + }, + { + name: "azure US government cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREUSGOVERNMENT", + }, + err: false, + expected: cloud.AzureGovernment, + }, + { + name: "azure US government cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREUSGOVERNMENTCLOUD", + }, + err: false, + expected: cloud.AzureGovernment, + }, + { + name: "AD authority URI provided", + bslCfg: map[string]string{ + BSLConfigActiveDirectoryAuthorityURI: "https://example.com", + }, + creds: map[string]string{ + CredentialKeyCloudName: "", + }, + err: false, + expected: publicCloudWithADURI, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg, err := getCloudConfiguration(c.bslCfg, c.creds) + require.Equal(t, c.err, err != nil) + if !c.err { + assert.Equal(t, c.expected, cfg) + } + }) + } +} + +func TestGetFromLocationConfigOrCredential(t *testing.T) { + // from cfg + cfg := map[string]string{ + "cfgkey": "value", + } + creds := map[string]string{} + cfgKey, credKey := "cfgkey", "credkey" + str := GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) + assert.Equal(t, "value", str) + + // from cred + cfg = map[string]string{} + creds = map[string]string{ + "credkey": "value", + } + str = GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) + assert.Equal(t, "value", str) +}