From ae160b02da910aca7433cfedd40460094620cea0 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 10 Sep 2021 09:37:26 +1000 Subject: [PATCH 01/83] ci: New dev/prod release workflow. --- .github/workflows/publish-binaries.yaml | 62 +++++++++ .github/workflows/publish-contracts.yaml | 41 ++++++ .../{release.yaml => publish-docker.yaml} | 41 +++--- .github/workflows/publish.yaml | 121 ------------------ .github/workflows/release-prod.yaml | 35 +++++ .github/workflows/release-rc.yaml | 43 +++++++ .github/workflows/test.yaml | 2 +- 7 files changed, 206 insertions(+), 139 deletions(-) create mode 100644 .github/workflows/publish-binaries.yaml create mode 100644 .github/workflows/publish-contracts.yaml rename .github/workflows/{release.yaml => publish-docker.yaml} (61%) delete mode 100644 .github/workflows/publish.yaml create mode 100644 .github/workflows/release-prod.yaml create mode 100644 .github/workflows/release-rc.yaml diff --git a/.github/workflows/publish-binaries.yaml b/.github/workflows/publish-binaries.yaml new file mode 100644 index 000000000..eab1e6e3a --- /dev/null +++ b/.github/workflows/publish-binaries.yaml @@ -0,0 +1,62 @@ +name: Release Binaries +on: + release: + types: [published] + +jobs: + add_binaries: + name: Add Binaries to release + runs-on: ubuntu-latest + env: + GOPATH: /home/runner/go + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.7 + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: Install modules + run: make install-tools + - name: Make binaries + run: make build-all-binaries + - name: Upload Dev + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-dev + asset_name: membrane-dev + asset_content_type: application/octet-stream + - name: Upload AWS + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-aws + asset_name: membrane-aws + asset_content_type: application/octet-stream + - name: Upload GCP + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-gcp + asset_name: membrane-gcp + asset_content_type: application/octet-stream + - name: Upload Digital Ocean + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-do + asset_name: membrane-do + asset_content_type: application/octet-stream \ No newline at end of file diff --git a/.github/workflows/publish-contracts.yaml b/.github/workflows/publish-contracts.yaml new file mode 100644 index 000000000..9f8d2b60b --- /dev/null +++ b/.github/workflows/publish-contracts.yaml @@ -0,0 +1,41 @@ +name: Release Contracts +on: + release: + types: [published] + +jobs: + # Bump the membrane version + contract_release: + name: Add contracts to release + runs-on: ubuntu-latest + env: + GOPATH: /home/runner/go + steps: + - name: Checkout + uses: actions/checkout@v2 + + # Add validate contract to the released contracts + - name: Download validate contract + run: | + mkdir -p ./contracts/validate + curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.6.1/validate/validate.proto --output ./contracts/validate/validate.proto + + # Tarball the contracts repository + - name: Archive Release + uses: thedoctor0/zip-release@master + with: + type: 'tar' + filename: 'contracts.tgz' + path: contracts + + # Upload contract tarball to the releases + - name: Upload Contracts + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./contracts.tgz + asset_name: contracts.tgz + asset_content_type: application/tar+gzip + \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/publish-docker.yaml similarity index 61% rename from .github/workflows/release.yaml rename to .github/workflows/publish-docker.yaml index e3984f989..aa22432a4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/publish-docker.yaml @@ -1,7 +1,7 @@ name: Release Docker images on: release: - types: [released] + types: [published] jobs: push_to_registry: name: Push Docker image to Docker Hub @@ -9,12 +9,6 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v2 - with: - token: ${{secrets.GOLANG_TOKEN}} - submodules: recursive - - name: Get Tag - id: tag - run: echo ::set-output name=TAG::$(echo $GITHUB_REF | cut -d / -f 3) # Setup docker build image - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -24,6 +18,19 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Sets the latest tag to rc-latest if release was a prerelease + - name: Set latest tag RC + if: "github.event.release.prerelease" + run: | + echo "latest_tag=rc-latest" >> $GITHUB_ENV + + # Uses latest tag if release was a production release + - name: Set latest tag Prod + if: "!github.event.release.prerelease" + run: | + echo "latest_tag=latest" >> $GITHUB_ENV + # Push development image - name: Push Local Static image to Docker Hub uses: docker/build-push-action@v2 @@ -32,8 +39,8 @@ jobs: file: ./pkg/providers/dev/dev.dockerfile push: true tags: | - nitricimages/membrane-local:latest - nitricimages/membrane-local:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-local:${{ env.latest_tag }} + nitricimages/membrane-local:${{ github.event.release.tag_name }} # Push AWS image - name: Push AWS Static image to Docker Hub uses: docker/build-push-action@v2 @@ -42,8 +49,8 @@ jobs: file: ./pkg/providers/aws/aws.dockerfile push: true tags: | - nitricimages/membrane-aws:latest - nitricimages/membrane-aws:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-aws:${{ env.latest_tag }} + nitricimages/membrane-aws:${{ github.event.release.tag_name }} # Push GCP image - name: Push GCP Static image to Docker Hub uses: docker/build-push-action@v2 @@ -52,8 +59,8 @@ jobs: file: ./pkg/providers/gcp/gcp.dockerfile push: true tags: | - nitricimages/membrane-gcp:latest - nitricimages/membrane-gcp:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-gcp:${{ env.latest_tag }} + nitricimages/membrane-gcp:${{ github.event.release.tag_name }} # Push Azure image - name: Push Azure Static image to Docker Hub uses: docker/build-push-action@v2 @@ -62,8 +69,8 @@ jobs: file: ./pkg/providers/azure/azure.dockerfile push: true tags: | - nitricimages/membrane-azure:latest - nitricimages/membrane-azure:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-azure:${{ env.latest_tag }} + nitricimages/membrane-azure:${{ github.event.release.tag_name }} # Push Digital Ocean image - name: Push Digital Ocean image to Docker Hub uses: docker/build-push-action@v2 @@ -72,5 +79,5 @@ jobs: file: ./pkg/providers/do/do.dockerfile push: true tags: | - nitricimages/membrane-do:latest - nitricimages/membrane-do:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-do:${{ env.latest_tag }} + nitricimages/membrane-do:${{ github.event.release.tag_name }} diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 53fd28521..000000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: Create Release -on: - pull_request: - types: [closed] - branches: - - 'main' -jobs: - # Bump the membrane version - version_bump: - if: github.event.pull_request.merged == true - name: Bump Version and Create Release - runs-on: ubuntu-latest - outputs: - version_id: ${{ steps.tag_version.outputs.new_tag }} - upload_url: ${{ steps.create_release.outputs.upload_url }} - steps: - - uses: actions/checkout@v2 - - name: Bump version and push tag - id: tag_version - uses: mathieudutour/github-tag-action@v5.5 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Create a GitHub release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: true - tag_name: ${{ steps.tag_version.outputs.new_tag }} - release_name: Release ${{ steps.tag_version.outputs.new_tag }} - body: ${{ steps.tag_version.outputs.changelog }} - - # Add membrane binaries to the release - add_binaries: - needs: version_bump - name: Add Binaries to release - runs-on: ubuntu-latest - env: - GOPATH: /home/runner/go - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Setup Go - uses: actions/setup-go@v2 - with: - go-version: 1.16.7 - - name: Install Protoc - uses: arduino/setup-protoc@v1 - - name: Install modules - run: make install-tools - - name: Make binaries - run: make build-all-binaries - - name: Upload Dev - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-dev - asset_name: membrane-dev - asset_content_type: application/octet-stream - - name: Upload AWS - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-aws - asset_name: membrane-aws - asset_content_type: application/octet-stream - - name: Upload GCP - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-gcp - asset_name: membrane-gcp - asset_content_type: application/octet-stream - - name: Upload Digital Ocean - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-do - asset_name: membrane-do - asset_content_type: application/octet-stream - - # Add contracts to release - add_contracts: - needs: version_bump - name: Add contracts to release - runs-on: ubuntu-latest - env: - GOPATH: /home/runner/go - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Download validate contract - run: | - mkdir -p ./contracts/validate - curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.6.1/validate/validate.proto --output ./contracts/validate/validate.proto - - name: Archive Release - uses: thedoctor0/zip-release@master - with: - type: 'tar' - filename: 'contracts.tgz' - path: ./contracts - - name: Upload Contracts - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./contracts.tgz - asset_name: contracts.tgz - asset_content_type: application/tar+gzip \ No newline at end of file diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml new file mode 100644 index 000000000..e6f88dee8 --- /dev/null +++ b/.github/workflows/release-prod.yaml @@ -0,0 +1,35 @@ +name: Production Release +on: + pull_request: + types: [closed] + branches: + - 'main' +jobs: + # Bump the membrane version + version_bump: + if: github.event.pull_request.merged == true + name: Bump Version and Create Release + runs-on: ubuntu-latest + outputs: + version_id: ${{ steps.tag_version.outputs.new_tag }} + upload_url: ${{ steps.create_release.outputs.upload_url }} + steps: + - uses: actions/checkout@v2 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + # Use GITHUB_TOKEN here to prevent further workflows + # generated on 'tag' action + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Create a GitHub release + id: create_release + uses: actions/create-release@v1 + env: + # Use NITRIC_BOT_TOKEN here to + # trigger release 'published' workflows + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + tag_name: ${{ steps.tag_version.outputs.new_tag }} + release_name: Release ${{ steps.tag_version.outputs.new_tag }} + body: ${{ steps.tag_version.outputs.changelog }} \ No newline at end of file diff --git a/.github/workflows/release-rc.yaml b/.github/workflows/release-rc.yaml new file mode 100644 index 000000000..4953d057b --- /dev/null +++ b/.github/workflows/release-rc.yaml @@ -0,0 +1,43 @@ +name: Release Candidate +on: + pull_request: + types: [closed] + branches: + - 'develop' +jobs: + # Bump the membrane version + version_bump: + if: github.event.pull_request.merged == true + name: Bump Version and Create Release + runs-on: ubuntu-latest + outputs: + version_id: ${{ steps.tag_version.outputs.new_tag }} + upload_url: ${{ steps.create_release.outputs.upload_url }} + steps: + - uses: actions/checkout@v2 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + # Don't commit tag + # this will be done as part of the release + dry_run: true + github_token: ${{ secrets.GITHUB_TOKEN }} + release_branches: main,develop + + - name: Calculate SHORT_SHA + id: vars + run: echo "::set-output name=sha_short::$(echo ${GITHUB_SHA} | cut -c1-8)" + + - name: Create a GitHub release + id: create_release + uses: actions/create-release@v1 + env: + # Use NITRIC_BOT_TOKEN here to + # trigger release 'published' workflows + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + prerelease: true + tag_name: ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.sha_short }} + release_name: Release ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.sha_short }} + body: ${{ steps.tag_version.outputs.changelog }}-rc.${{ steps.vars.outputs.sha_short }} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4d00cc462..bc3dc2edd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,9 +4,9 @@ on: push: branches: - main + - develop pull_request: - jobs: test: runs-on: ubuntu-latest From 4267432a09e215fbb239eff4a3590889ecff1adc Mon Sep 17 00:00:00 2001 From: medgar Date: Fri, 10 Sep 2021 10:44:41 +1000 Subject: [PATCH 02/83] chore:added dynamodb, firestore test containers --- README.md | 23 +----- makefile | 7 -- .../document/dynamodb/dynamodb_test.go | 78 ++++++++++++++----- .../document/firestore/firestore_test.go | 76 +++++++++++++----- 4 files changed, 116 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index c5176ee63..c807466f4 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,9 @@ SDKs are available for many languages, providing an idiomatic wrapper around the - [Node.js](https://github.com/nitrictech/node-sdk) - [Python](https://github.com/nitrictech/python-sdk) - - [Go](https://github.com/nitrictech/go-sdk) - [Java](https://github.com/nitrictech/java-sdk) - - [PHP](https://github.com/nitrictech/php-sdk) - - [.NET](https://github.com/nitrictech/dotnet-sdk) - + - [Go](https://github.com/nitrictech/go-sdk) + > If you have additional languages you would like support for let us know in the issues, we also welcome community contribtions for new language support. ## Architecture @@ -83,8 +81,6 @@ provider as an alternative to the fixed set of plugins in the static membranes. - Make - Docker - Google Protocol Buffers Compiler - - Google Cloud SDK (for Firestore testing) - - JRE (for DynamoDB testing) ### Getting Started @@ -93,26 +89,11 @@ provider as an alternative to the fixed set of plugins in the static membranes. make install-tools ``` -#### Install integration testing tools -```bash -make install-test-tools -``` - ##### Install Protocol Buffers Download the Google Protobuf Compiler (standalone binary called `protoc`) from https://github.com/protocolbuffers/protobuf and add it to your $PATH. > On MacOS with Homebrew, you can run `brew install protobuf` -##### Install Google Cloud SDK -Install the Google Cloud SDK following in the instructions at: https://cloud.google.com/sdk/docs/install - -#### Install JRE -Install a Java Runtime Environment (JRE) version 11 or later for your OS. For example on Ubuntu Linux run: - -```bash -sudo apt-get install openjdk-11-jdk -``` - ### Run unit tests ```bash make tests diff --git a/makefile b/makefile index 42977531a..943ce0b2f 100644 --- a/makefile +++ b/makefile @@ -18,13 +18,6 @@ install-tools: install @echo Installing tools from tools.go @cat ./tools/tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go get % -# Install integration testing tools -install-test-tools: - @wget https://s3.us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.tar.gz - @sudo mkdir -p /usr/local/dynamodb - @sudo tar -xf dynamodb_local_latest.tar.gz -C /usr/local/dynamodb - @rm dynamodb_local_latest.tar.gz - clean: @rm -rf ./bin/ @rm -rf ./lib/ diff --git a/tests/plugins/document/dynamodb/dynamodb_test.go b/tests/plugins/document/dynamodb/dynamodb_test.go index 2a22ea6db..656499192 100644 --- a/tests/plugins/document/dynamodb/dynamodb_test.go +++ b/tests/plugins/document/dynamodb/dynamodb_test.go @@ -16,9 +16,12 @@ package dynamodb_service_test import ( "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/document/dynamodb" "os" "os/exec" + "strings" + "syscall" + + dynamodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/dynamodb" test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" @@ -28,6 +31,10 @@ import ( "github.com/aws/aws-sdk-go/service/dynamodb" ) +const shell = "/bin/sh" +const containerName = "dynamodb-nitric" +const port = "8000" + var _ = Describe("DynamoDb", func() { defer GinkgoRecover() @@ -36,7 +43,7 @@ var _ = Describe("DynamoDb", func() { os.Setenv("AWS_REGION", "X") // Start Local DynamoDB - dynaCmd := startDynamoProcess() + startDynamoContainer() // Create DynamoDB client db := createDynamoClient() @@ -57,7 +64,7 @@ var _ = Describe("DynamoDb", func() { }) AfterSuite(func() { - stopDynamoProcess(dynaCmd) + stopDynamoContainer() }) docPlugin, err := dynamodb_service.NewWithClient(db) @@ -71,33 +78,64 @@ var _ = Describe("DynamoDb", func() { test.QueryTests(docPlugin) }) -func startDynamoProcess() *exec.Cmd { - // Start Local DynamoDB +func startDynamoContainer() { + // Run dynamodb container args := []string{ - "-Djava.library.path=/usr/local/dynamodb/DynamoDBLocal_lib", - "-jar", - "/usr/local/dynamodb/DynamoDBLocal.jar", - "-inMemory", + "docker", + "run", + "-d", + "-p " + port + ":" + port, + "--name " + containerName, + "amazon/dynamodb-local:latest", } - cmd := exec.Command("/usr/bin/java", args[:]...) - if err := cmd.Start(); err != nil { - panic(fmt.Sprintf("Error starting Local DynamoDB %v : %v", cmd, err)) + + cmd := exec.Command("/bin/sh", "-c", strings.Join(args[:], " ")) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error running DynamoDB Image %v : %v \n", cmd, err) + panic(fmt.Sprintf("Error running DynamoDB Image %v : %v", cmd, err)) } - fmt.Printf("Started Local DynamoDB (PID %v) and loading data...\n", cmd.Process.Pid) - return cmd + // Makes process killable + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} } -func stopDynamoProcess(cmd *exec.Cmd) { - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("failed to kill DynamoDB %v : %v \n", cmd.Process.Pid, err) +func stopDynamoContainer() { + // clean up + stopArgs := []string{ + "docker", + "container", + "stop", + containerName, + } + + stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) + + if err := stopCmd.Run(); err != nil { + fmt.Printf("Error stopping DynamoDB container %v : %v \n", stopCmd, err) + panic(fmt.Sprintf("Error stopping DynamoDB container %v : %v", stopCmd, err)) + } + + removeArgs := []string{ + "docker", + "container", + "rm", + containerName, + } + + removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) + + if err := removeCmd.Run(); err != nil { + fmt.Printf("Error removing DynamoDB container %v : %v \n", removeCmd, err) + panic(fmt.Sprintf("Error removing DynamoDB container %v : %v", removeCmd, err)) } } func createDynamoClient() *dynamodb.DynamoDB { sess := session.Must(session.NewSession(&aws.Config{ Region: aws.String("x"), - Endpoint: aws.String("http://127.0.0.1:8000"), + Endpoint: aws.String("http://127.0.0.1:" + port), })) return dynamodb.New(sess) @@ -132,7 +170,7 @@ func createTable(db *dynamodb.DynamoDB, tableName string) { TableName: aws.String(tableName), Tags: []*dynamodb.Tag{ { - Key: aws.String("x-nitric-name"), + Key: aws.String("x-nitric-name"), Value: aws.String(tableName), }, }, @@ -152,4 +190,4 @@ func deleteTable(db *dynamodb.DynamoDB, tableName string) { if err != nil { panic(fmt.Sprintf("Error calling DeleteTable: %s", err)) } -} \ No newline at end of file +} diff --git a/tests/plugins/document/firestore/firestore_test.go b/tests/plugins/document/firestore/firestore_test.go index 90d249069..9d5dddf4c 100644 --- a/tests/plugins/document/firestore/firestore_test.go +++ b/tests/plugins/document/firestore/firestore_test.go @@ -17,41 +17,74 @@ package firestore_service_test import ( "context" "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/document/firestore" "os" "os/exec" + "strings" "syscall" + firestore_service "github.com/nitric-dev/membrane/pkg/plugins/document/firestore" + "cloud.google.com/go/firestore" test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" ) -func startFirestoreProcess() *exec.Cmd { - // Start Local DynamoDB - os.Setenv("FIRESTORE_EMULATOR_HOST", "localhost:8080") +const shell = "/bin/sh" +const containerName = "firestore-nitric" +const port = "8080" - // Create Firestore Process +func startFirestoreContainer() { + // Run dynamodb container args := []string{ - "beta", - "emulators", - "firestore", - "start", - "--host-port=localhost:8080", + "docker", + "run", + "-d", + "-p " + port + ":" + port, + "--env \"FIRESTORE_PROJECT_ID=dummy-project-id\"", + "--name " + containerName, + "mtlynch/firestore-emulator-docker", } - cmd := exec.Command("gcloud", args[:]...) - if err := cmd.Start(); err != nil { - panic(fmt.Sprintf("Error starting Firestore Emulator %v : %v", cmd, err)) + + cmd := exec.Command("/bin/sh", "-c", strings.Join(args[:], " ")) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error running Firestore Image %v : %v \n", cmd, err) + panic(fmt.Sprintf("Error running Firestore Image %v : %v", cmd, err)) } + // Makes process killable cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - - return cmd } -func stopFirestoreProcess(cmd *exec.Cmd) { - if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { - fmt.Printf("\nFailed to kill Firestore %v : %v \n", cmd.Process.Pid, err) +func stopFirestoreContainer() { + // clean up + stopArgs := []string{ + "docker", + "container", + "stop", + containerName, + } + + stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) + + if err := stopCmd.Run(); err != nil { + fmt.Printf("Error stopping Firestore container %v : %v \n", stopCmd, err) + panic(fmt.Sprintf("Error stopping Firestore container %v : %v", stopCmd, err)) + } + + removeArgs := []string{ + "docker", + "container", + "rm", + containerName, + } + + removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) + + if err := removeCmd.Run(); err != nil { + fmt.Printf("Error removing Firestore container %v : %v \n", removeCmd, err) + panic(fmt.Sprintf("Error removing Firestore container %v : %v", removeCmd, err)) } } @@ -68,14 +101,17 @@ func createFirestoreClient(ctx context.Context) *firestore.Client { var _ = Describe("Firestore", func() { defer GinkgoRecover() + // Start Local DynamoDB + os.Setenv("FIRESTORE_EMULATOR_HOST", "localhost:"+port) + // Start Firestore Emulator - firestoreCmd := startFirestoreProcess() + startFirestoreContainer() ctx := context.Background() db := createFirestoreClient(ctx) AfterSuite(func() { - stopFirestoreProcess(firestoreCmd) + stopFirestoreContainer() }) docPlugin, err := firestore_service.NewWithClient(db, ctx) From 5e2bad34bb1801d6e423ad8e1afe7ec9d025ccd3 Mon Sep 17 00:00:00 2001 From: medgar Date: Fri, 10 Sep 2021 11:12:52 +1000 Subject: [PATCH 03/83] chore:sonar cloud updates --- tests/plugins/docker.go | 70 ++++++++++++++++ .../document/dynamodb/dynamodb_test.go | 71 +++------------- .../document/firestore/firestore_test.go | 72 +++-------------- .../plugins/document/mongodb/mongodb_test.go | 81 ++++--------------- 4 files changed, 110 insertions(+), 184 deletions(-) create mode 100644 tests/plugins/docker.go diff --git a/tests/plugins/docker.go b/tests/plugins/docker.go new file mode 100644 index 000000000..98b1f568f --- /dev/null +++ b/tests/plugins/docker.go @@ -0,0 +1,70 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugins + +import ( + "fmt" + "os" + "os/exec" + "strings" + "syscall" +) + +const shell = "/bin/sh" + +func StartContainer(containerName string, args []string) { + + cmd := exec.Command(shell, "-c", strings.Join(args[:], " ")) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error running %s Image %v : %v \n", containerName, cmd, err) + panic(fmt.Sprintf("Error running %s Image %v : %v", containerName, cmd, err)) + } + + // Makes process killable + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func StopContainer(containerName string) { + // clean up + stopArgs := []string{ + "docker", + "container", + "stop", + containerName, + } + + stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) + + if err := stopCmd.Run(); err != nil { + fmt.Printf("Error stopping %s container %v : %v \n", containerName, stopCmd, err) + panic(fmt.Sprintf("Error stopping Firestore container %v : %v", stopCmd, err)) + } + + removeArgs := []string{ + "docker", + "container", + "rm", + containerName, + } + + removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) + + if err := removeCmd.Run(); err != nil { + fmt.Printf("Error removing %s container %v : %v \n", containerName, removeCmd, err) + panic(fmt.Sprintf("Error removing Firestore container %v : %v", removeCmd, err)) + } +} diff --git a/tests/plugins/document/dynamodb/dynamodb_test.go b/tests/plugins/document/dynamodb/dynamodb_test.go index 656499192..1a4ccb16e 100644 --- a/tests/plugins/document/dynamodb/dynamodb_test.go +++ b/tests/plugins/document/dynamodb/dynamodb_test.go @@ -17,12 +17,10 @@ package dynamodb_service_test import ( "fmt" "os" - "os/exec" - "strings" - "syscall" dynamodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/dynamodb" + "github.com/nitric-dev/membrane/tests/plugins" test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" @@ -43,7 +41,16 @@ var _ = Describe("DynamoDb", func() { os.Setenv("AWS_REGION", "X") // Start Local DynamoDB - startDynamoContainer() + // Run dynamodb container + args := []string{ + "docker", + "run", + "-d", + "-p " + port + ":" + port, + "--name " + containerName, + "amazon/dynamodb-local:latest", + } + plugins.StartContainer(containerName, args) // Create DynamoDB client db := createDynamoClient() @@ -64,7 +71,7 @@ var _ = Describe("DynamoDb", func() { }) AfterSuite(func() { - stopDynamoContainer() + plugins.StopContainer(containerName) }) docPlugin, err := dynamodb_service.NewWithClient(db) @@ -78,60 +85,6 @@ var _ = Describe("DynamoDb", func() { test.QueryTests(docPlugin) }) -func startDynamoContainer() { - // Run dynamodb container - args := []string{ - "docker", - "run", - "-d", - "-p " + port + ":" + port, - "--name " + containerName, - "amazon/dynamodb-local:latest", - } - - cmd := exec.Command("/bin/sh", "-c", strings.Join(args[:], " ")) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - fmt.Printf("Error running DynamoDB Image %v : %v \n", cmd, err) - panic(fmt.Sprintf("Error running DynamoDB Image %v : %v", cmd, err)) - } - - // Makes process killable - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} -} - -func stopDynamoContainer() { - // clean up - stopArgs := []string{ - "docker", - "container", - "stop", - containerName, - } - - stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) - - if err := stopCmd.Run(); err != nil { - fmt.Printf("Error stopping DynamoDB container %v : %v \n", stopCmd, err) - panic(fmt.Sprintf("Error stopping DynamoDB container %v : %v", stopCmd, err)) - } - - removeArgs := []string{ - "docker", - "container", - "rm", - containerName, - } - - removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) - - if err := removeCmd.Run(); err != nil { - fmt.Printf("Error removing DynamoDB container %v : %v \n", removeCmd, err) - panic(fmt.Sprintf("Error removing DynamoDB container %v : %v", removeCmd, err)) - } -} - func createDynamoClient() *dynamodb.DynamoDB { sess := session.Must(session.NewSession(&aws.Config{ Region: aws.String("x"), diff --git a/tests/plugins/document/firestore/firestore_test.go b/tests/plugins/document/firestore/firestore_test.go index 9d5dddf4c..bfb863316 100644 --- a/tests/plugins/document/firestore/firestore_test.go +++ b/tests/plugins/document/firestore/firestore_test.go @@ -18,13 +18,11 @@ import ( "context" "fmt" "os" - "os/exec" - "strings" - "syscall" firestore_service "github.com/nitric-dev/membrane/pkg/plugins/document/firestore" "cloud.google.com/go/firestore" + "github.com/nitric-dev/membrane/tests/plugins" test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" ) @@ -33,61 +31,6 @@ const shell = "/bin/sh" const containerName = "firestore-nitric" const port = "8080" -func startFirestoreContainer() { - // Run dynamodb container - args := []string{ - "docker", - "run", - "-d", - "-p " + port + ":" + port, - "--env \"FIRESTORE_PROJECT_ID=dummy-project-id\"", - "--name " + containerName, - "mtlynch/firestore-emulator-docker", - } - - cmd := exec.Command("/bin/sh", "-c", strings.Join(args[:], " ")) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - fmt.Printf("Error running Firestore Image %v : %v \n", cmd, err) - panic(fmt.Sprintf("Error running Firestore Image %v : %v", cmd, err)) - } - - // Makes process killable - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} -} - -func stopFirestoreContainer() { - // clean up - stopArgs := []string{ - "docker", - "container", - "stop", - containerName, - } - - stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) - - if err := stopCmd.Run(); err != nil { - fmt.Printf("Error stopping Firestore container %v : %v \n", stopCmd, err) - panic(fmt.Sprintf("Error stopping Firestore container %v : %v", stopCmd, err)) - } - - removeArgs := []string{ - "docker", - "container", - "rm", - containerName, - } - - removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) - - if err := removeCmd.Run(); err != nil { - fmt.Printf("Error removing Firestore container %v : %v \n", removeCmd, err) - panic(fmt.Sprintf("Error removing Firestore container %v : %v", removeCmd, err)) - } -} - func createFirestoreClient(ctx context.Context) *firestore.Client { client, err := firestore.NewClient(ctx, "test") if err != nil { @@ -105,13 +48,22 @@ var _ = Describe("Firestore", func() { os.Setenv("FIRESTORE_EMULATOR_HOST", "localhost:"+port) // Start Firestore Emulator - startFirestoreContainer() + args := []string{ + "docker", + "run", + "-d", + "-p " + port + ":" + port, + "--env \"FIRESTORE_PROJECT_ID=dummy-project-id\"", + "--name " + containerName, + "mtlynch/firestore-emulator-docker", + } + plugins.StartContainer(containerName, args) ctx := context.Background() db := createFirestoreClient(ctx) AfterSuite(func() { - stopFirestoreContainer() + plugins.StopContainer(containerName) }) docPlugin, err := firestore_service.NewWithClient(db, ctx) diff --git a/tests/plugins/document/mongodb/mongodb_test.go b/tests/plugins/document/mongodb/mongodb_test.go index 94df0725e..93337a6d4 100644 --- a/tests/plugins/document/mongodb/mongodb_test.go +++ b/tests/plugins/document/mongodb/mongodb_test.go @@ -17,73 +17,16 @@ package mongodb_service_test import ( "context" "fmt" - "os" - "os/exec" - "strings" - "syscall" mongodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/mongodb" + "github.com/nitric-dev/membrane/tests/plugins" test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) -var shell = "/bin/sh" - -func startMongoImage() *exec.Cmd { - // Run mongodb container - args := []string{ - "docker", - "run", - "-d", - "-p 27017-27019:27017-27019", - "--name mongodb-nitric", - "mongo:4.0", - } - - cmd := exec.Command("/bin/sh", "-c", strings.Join(args[:], " ")) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - panic(fmt.Sprintf("Error running MongoDB Image %v : %v", cmd, err)) - } - - // Makes process killable - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - - return cmd -} - -func stopMongoImage(cmd *exec.Cmd) { - - // clean up - stopArgs := []string{ - "docker", - "container", - "stop", - "mongodb-nitric", - } - - stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) - - if err := stopCmd.Run(); err != nil { - panic(fmt.Sprintf("Error stopping MongoDB container %v : %v", cmd, err)) - } - - removeArgs := []string{ - "docker", - "container", - "rm", - "mongodb-nitric", - } - - removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) - - if err := removeCmd.Run(); err != nil { - panic(fmt.Sprintf("Error removing MongoDB container %v : %v", cmd, err)) - } -} +const containerName = "mongodb-nitric" func createMongoClient(ctx context.Context) (*mongo.Client, error) { clientOptions := options.Client().ApplyURI("mongodb://localhost:27017").SetDirect(true) @@ -92,7 +35,7 @@ func createMongoClient(ctx context.Context) (*mongo.Client, error) { if clientError != nil { return nil, fmt.Errorf("mongodb error creating client: %v", clientError) } - + connectError := client.Connect(ctx) if connectError != nil { @@ -100,11 +43,11 @@ func createMongoClient(ctx context.Context) (*mongo.Client, error) { } pingError := client.Ping(ctx, nil) - + if pingError != nil { return nil, fmt.Errorf("mongodb unable to connect: %v", pingError) } - + return client, nil } @@ -112,10 +55,18 @@ var _ = Describe("MongoDB", func() { defer GinkgoRecover() // Start Mongo - mongoCmd := startMongoImage() + args := []string{ + "docker", + "run", + "-d", + "-p 27017-27019:27017-27019", + "--name " + containerName, + "mongo:4.0", + } + plugins.StartContainer(containerName, args) AfterSuite(func() { - stopMongoImage(mongoCmd) + plugins.StopContainer(containerName) }) ctx := context.Background() @@ -134,7 +85,7 @@ var _ = Describe("MongoDB", func() { } test.GetTests(docPlugin) - test.SetTests(docPlugin) + test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) }) From 9043bd280652485d5143091402a53c16d289b2b2 Mon Sep 17 00:00:00 2001 From: medgar Date: Fri, 10 Sep 2021 13:46:28 +1000 Subject: [PATCH 04/83] chore:fix test.yaml --- .github/workflows/test.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4d00cc462..2c47cfe06 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -58,18 +58,5 @@ jobs: go-version: 1.16.7 - name: Install Protoc uses: arduino/setup-protoc@v1 - - name: Install modules - run: make install-tools - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - java-version: '11' - distribution: 'adopt' - - name: Setup local DynamoDB - run: make install-test-tools - - name: Setup Cloud SDK - uses: google-github-actions/setup-gcloud@master - - name: Install Google Cloud SDK components - run: yes | gcloud components install beta cloud-firestore-emulator - name: Run Integration Tests run: make test-integration \ No newline at end of file From ed273a97c0eef22c2e9f5ebceca6f2b3b8563811 Mon Sep 17 00:00:00 2001 From: medgar Date: Fri, 10 Sep 2021 14:09:38 +1000 Subject: [PATCH 05/83] chore:dynamo action fix --- tests/plugins/document/dynamodb/dynamodb_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugins/document/dynamodb/dynamodb_test.go b/tests/plugins/document/dynamodb/dynamodb_test.go index 1a4ccb16e..dcb5f9a01 100644 --- a/tests/plugins/document/dynamodb/dynamodb_test.go +++ b/tests/plugins/document/dynamodb/dynamodb_test.go @@ -88,7 +88,7 @@ var _ = Describe("DynamoDb", func() { func createDynamoClient() *dynamodb.DynamoDB { sess := session.Must(session.NewSession(&aws.Config{ Region: aws.String("x"), - Endpoint: aws.String("http://127.0.0.1:" + port), + Endpoint: aws.String("http://localhost:" + port), })) return dynamodb.New(sess) From 925738448cabee5520d3cbfdb03ee4e5dfcee4b2 Mon Sep 17 00:00:00 2001 From: medgar Date: Fri, 10 Sep 2021 14:37:13 +1000 Subject: [PATCH 06/83] chore:debugging dynamo-local --- .../document/dynamodb/dynamodb_test.go | 86 +++++++++---------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/tests/plugins/document/dynamodb/dynamodb_test.go b/tests/plugins/document/dynamodb/dynamodb_test.go index dcb5f9a01..15112e0a4 100644 --- a/tests/plugins/document/dynamodb/dynamodb_test.go +++ b/tests/plugins/document/dynamodb/dynamodb_test.go @@ -18,10 +18,6 @@ import ( "fmt" "os" - dynamodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/dynamodb" - - "github.com/nitric-dev/membrane/tests/plugins" - test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" "github.com/aws/aws-sdk-go/aws" @@ -42,47 +38,47 @@ var _ = Describe("DynamoDb", func() { // Start Local DynamoDB // Run dynamodb container - args := []string{ - "docker", - "run", - "-d", - "-p " + port + ":" + port, - "--name " + containerName, - "amazon/dynamodb-local:latest", - } - plugins.StartContainer(containerName, args) - - // Create DynamoDB client - db := createDynamoClient() - - BeforeEach(func() { - // Table names suffixed with 7 alphanumeric chars to match pulumi deployment. - createTable(db, "customers-1111111") - createTable(db, "users-1111111") - createTable(db, "items-1111111") - createTable(db, "parentItems-1111111") - }) - - AfterEach(func() { - deleteTable(db, "customers-1111111") - deleteTable(db, "users-1111111") - deleteTable(db, "items-1111111") - deleteTable(db, "parentItems-1111111") - }) - - AfterSuite(func() { - plugins.StopContainer(containerName) - }) - - docPlugin, err := dynamodb_service.NewWithClient(db) - if err != nil { - panic(err) - } - - test.GetTests(docPlugin) - test.SetTests(docPlugin) - test.DeleteTests(docPlugin) - test.QueryTests(docPlugin) + // args := []string{ + // "docker", + // "run", + // "-d", + // "-p " + port + ":" + port, + // "--name " + containerName, + // "amazon/dynamodb-local:latest", + // } + // plugins.StartContainer(containerName, args) + + // // Create DynamoDB client + // db := createDynamoClient() + + // BeforeEach(func() { + // // Table names suffixed with 7 alphanumeric chars to match pulumi deployment. + // createTable(db, "customers-1111111") + // createTable(db, "users-1111111") + // createTable(db, "items-1111111") + // createTable(db, "parentItems-1111111") + // }) + + // AfterEach(func() { + // deleteTable(db, "customers-1111111") + // deleteTable(db, "users-1111111") + // deleteTable(db, "items-1111111") + // deleteTable(db, "parentItems-1111111") + // }) + + // AfterSuite(func() { + // plugins.StopContainer(containerName) + // }) + + // docPlugin, err := dynamodb_service.NewWithClient(db) + // if err != nil { + // panic(err) + // } + + // test.GetTests(docPlugin) + // test.SetTests(docPlugin) + // test.DeleteTests(docPlugin) + // test.QueryTests(docPlugin) }) func createDynamoClient() *dynamodb.DynamoDB { From 3f47d8537a1bc5b1f6bd6254b3cec7e3d6bf10c4 Mon Sep 17 00:00:00 2001 From: medgar Date: Fri, 10 Sep 2021 15:24:35 +1000 Subject: [PATCH 07/83] chore:add dynamo startup test --- .../document/dynamodb/dynamodb_test.go | 108 +++++++++++------- 1 file changed, 68 insertions(+), 40 deletions(-) diff --git a/tests/plugins/document/dynamodb/dynamodb_test.go b/tests/plugins/document/dynamodb/dynamodb_test.go index 15112e0a4..04f5dfd6f 100644 --- a/tests/plugins/document/dynamodb/dynamodb_test.go +++ b/tests/plugins/document/dynamodb/dynamodb_test.go @@ -17,6 +17,12 @@ package dynamodb_service_test import ( "fmt" "os" + "time" + + dynamodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/dynamodb" + + "github.com/nitric-dev/membrane/tests/plugins" + test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" @@ -38,58 +44,80 @@ var _ = Describe("DynamoDb", func() { // Start Local DynamoDB // Run dynamodb container - // args := []string{ - // "docker", - // "run", - // "-d", - // "-p " + port + ":" + port, - // "--name " + containerName, - // "amazon/dynamodb-local:latest", - // } - // plugins.StartContainer(containerName, args) + args := []string{ + "docker", + "run", + "-d", + "-p " + port + ":" + port, + "--name " + containerName, + "amazon/dynamodb-local:latest", + } + plugins.StartContainer(containerName, args) // // Create DynamoDB client - // db := createDynamoClient() - - // BeforeEach(func() { - // // Table names suffixed with 7 alphanumeric chars to match pulumi deployment. - // createTable(db, "customers-1111111") - // createTable(db, "users-1111111") - // createTable(db, "items-1111111") - // createTable(db, "parentItems-1111111") - // }) - - // AfterEach(func() { - // deleteTable(db, "customers-1111111") - // deleteTable(db, "users-1111111") - // deleteTable(db, "items-1111111") - // deleteTable(db, "parentItems-1111111") - // }) - - // AfterSuite(func() { - // plugins.StopContainer(containerName) - // }) - - // docPlugin, err := dynamodb_service.NewWithClient(db) - // if err != nil { - // panic(err) - // } - - // test.GetTests(docPlugin) - // test.SetTests(docPlugin) - // test.DeleteTests(docPlugin) - // test.QueryTests(docPlugin) + db := createDynamoClient() + + testConnection(db) + + BeforeEach(func() { + // Table names suffixed with 7 alphanumeric chars to match pulumi deployment. + createTable(db, "customers-1111111") + createTable(db, "users-1111111") + createTable(db, "items-1111111") + createTable(db, "parentItems-1111111") + }) + + AfterEach(func() { + deleteTable(db, "customers-1111111") + deleteTable(db, "users-1111111") + deleteTable(db, "items-1111111") + deleteTable(db, "parentItems-1111111") + }) + + AfterSuite(func() { + plugins.StopContainer(containerName) + }) + + docPlugin, err := dynamodb_service.NewWithClient(db) + if err != nil { + panic(err) + } + + test.GetTests(docPlugin) + test.SetTests(docPlugin) + test.DeleteTests(docPlugin) + test.QueryTests(docPlugin) }) func createDynamoClient() *dynamodb.DynamoDB { sess := session.Must(session.NewSession(&aws.Config{ Region: aws.String("x"), - Endpoint: aws.String("http://localhost:" + port), + Endpoint: aws.String("http://localhost:8000"), })) return dynamodb.New(sess) } +func testConnection(db *dynamodb.DynamoDB) { + input := &dynamodb.ListTablesInput{} + + if _, err := db.ListTables(input); err != nil { + // Wait for Java DynamoDB process to get started + time.Sleep(2 * time.Second) + + if _, err := db.ListTables(input); err != nil { + time.Sleep(4 * time.Second) + + if _, err := db.ListTables(input); err != nil { + fmt.Printf("DynamoDB connection error: %v \n", err) + panic(err) + } + } else { + return + } + } +} + func createTable(db *dynamodb.DynamoDB, tableName string) { input := &dynamodb.CreateTableInput{ AttributeDefinitions: []*dynamodb.AttributeDefinition{ From a4f269d587f3024bfefca58e710a868565ac2606 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Mon, 13 Sep 2021 08:47:16 +1000 Subject: [PATCH 08/83] ci: Add workflow for triggering automatic generation of base SDKs. --- .github/workflows/generate-sdks.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/generate-sdks.yaml diff --git a/.github/workflows/generate-sdks.yaml b/.github/workflows/generate-sdks.yaml new file mode 100644 index 000000000..2b55be325 --- /dev/null +++ b/.github/workflows/generate-sdks.yaml @@ -0,0 +1,19 @@ +name: Generate SDKs +on: + release: + types: [published] + +jobs: + # Bump the membrane version + generate_sdks: + name: Signal base SDK repo for auto regen + runs-on: ubuntu-latest + steps: + - name: Signal base SDK repo + uses: peter-evans/repository-dispatch@v1 + with: + token: ${{ secrets.NITRIC_BOT_TOKEN }} + repository: nitrictech/base-sdk + event-type: generate + client-payload: '{ "prerelease": ${{ github.event.release.prerelease }}, "version": "${{ github.event.release.tag_name }}" }' + \ No newline at end of file From 23759ace61a114e7c9d485f837a4d479c2e827af Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Mon, 13 Sep 2021 09:40:14 +1000 Subject: [PATCH 09/83] chore: use multiline string for payload. --- .github/workflows/generate-sdks.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate-sdks.yaml b/.github/workflows/generate-sdks.yaml index 2b55be325..d81061a66 100644 --- a/.github/workflows/generate-sdks.yaml +++ b/.github/workflows/generate-sdks.yaml @@ -15,5 +15,9 @@ jobs: token: ${{ secrets.NITRIC_BOT_TOKEN }} repository: nitrictech/base-sdk event-type: generate - client-payload: '{ "prerelease": ${{ github.event.release.prerelease }}, "version": "${{ github.event.release.tag_name }}" }' + client-payload: > + { + "prerelease": ${{ github.event.release.prerelease }}, + "tag_name": "${{ github.event.release.tag_name }}" + } \ No newline at end of file From 6dc6bce5b4914723ccece7ac3c4eb8aae6c56d96 Mon Sep 17 00:00:00 2001 From: medgar Date: Thu, 9 Sep 2021 18:45:07 +1000 Subject: [PATCH 10/83] feat: add grpc errors --- contracts/proto/error/v1/error.proto | 34 ++++++ go.sum | 51 +-------- pkg/adapters/grpc/errors.go | 108 +++++++++++++++++- pkg/adapters/grpc/errors_test.go | 96 +++++++++++++++- pkg/adapters/grpc/event_grpc_test.go | 2 - pkg/plugins/document/boltdb/boltdb.go | 16 ++- pkg/plugins/document/dynamodb/dynamodb.go | 16 ++- pkg/plugins/document/firestore/firestore.go | 16 ++- pkg/plugins/document/mongodb/mongodb.go | 87 +++++++------- pkg/plugins/document/plugin.go | 16 +-- pkg/plugins/errors/plugin_error.go | 51 +++------ pkg/plugins/events/dev/eventing.go | 7 +- pkg/plugins/events/event.go | 4 +- pkg/plugins/events/pubsub/pubsub.go | 7 +- pkg/plugins/events/sns/sns.go | 7 +- pkg/plugins/queue/dev/queue.go | 26 ++++- pkg/plugins/queue/plugin.go | 4 +- pkg/plugins/queue/pubsub/pubsub.go | 21 +++- pkg/plugins/queue/sqs/sqs.go | 19 ++- pkg/plugins/queue/task.go | 6 +- pkg/plugins/secret/dev/dev.go | 14 ++- .../secret/secret_manager/secret_manager.go | 14 ++- pkg/plugins/secret/secret_suite_test.go | 2 +- pkg/plugins/secret/secret_test.go | 5 +- .../secret/secrets_manager/secrets_manager.go | 14 ++- pkg/plugins/secret/types.go | 6 +- pkg/plugins/storage/boltdb/storage.go | 20 ++-- pkg/plugins/storage/s3/s3.go | 19 ++- pkg/plugins/storage/storage/storage.go | 19 ++- 29 files changed, 492 insertions(+), 215 deletions(-) create mode 100644 contracts/proto/error/v1/error.proto diff --git a/contracts/proto/error/v1/error.proto b/contracts/proto/error/v1/error.proto new file mode 100644 index 000000000..f8abd9418 --- /dev/null +++ b/contracts/proto/error/v1/error.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; +package nitric.error.v1; + +import "google/protobuf/struct.proto"; + +// protoc plugin options for code generation +option go_package = "nitric/v1;v1"; +option java_package = "io.nitric.proto.error.v1"; +option java_multiple_files = true; +option java_outer_classname = "Errors"; +option php_namespace = "Nitric\\Proto\\Error\\V1"; +option csharp_namespace = "Nitric.Proto.Error.v1"; + +message ErrorScope { + // The API service invoked, e.g. 'Service.Method'. + string service = 1; + + // The plugin method invoked, e.g. 'PluginService.Method'. + string plugin = 2; + + // The plugin method arguments, ensure only non-sensitive data is specified. + map args = 3; +} + +message ErrorDetails { + // The developer error message, explaining the error and ideally solution. + string message = 1; + + // The error details. + google.protobuf.Struct details = 2; + + // The scope of the error. + ErrorScope scope = 3; +} \ No newline at end of file diff --git a/go.sum b/go.sum index 9661fc7fb..bf43d2d59 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,8 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.5.0 h1:4qNItsmc4GP6UOZPGemmHY4ZfPofVhcaKXsYw9wm9oA= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= @@ -40,7 +38,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v51.3.0+incompatible h1:Y3wR7C5Sj0nZG3VhkePF5hK7zNCS5yeImN/k2CWB+u8= github.com/Azure/azure-sdk-for-go v51.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -60,9 +57,7 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -80,19 +75,13 @@ github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -104,9 +93,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= github.com/envoyproxy/protoc-gen-validate v0.6.1/go.mod h1:txg5va2Qkip90uYoSKH+nkAAmXrb2j3iq4FLwdrCbXQ= @@ -115,10 +102,8 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -148,7 +133,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -182,18 +166,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/addlicense v0.0.0-20210428195630-6d92264d7170 h1:jLUa4MO3autxlRJmC4KubeE5QGIb5JqW9oEaqYTb/fA= -github.com/google/addlicense v0.0.0-20210428195630-6d92264d7170/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= -github.com/google/addlicense v0.0.0-20210727174409-874627749a46 h1:1locMH9PVZH3LXvogcvdTxf2/9J4YT/9W3BSXrTN4/U= -github.com/google/addlicense v0.0.0-20210727174409-874627749a46/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= -github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b h1:KwI0NOpYd3rzKojfjeRerF7rzjeTwvJARVsgGf5TWmY= -github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= -github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab h1:+qfOxKbnAqDNCoFUNHxudKs8Z14T5EBYntAeWIeI1eA= -github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= github.com/google/addlicense v1.0.0/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -222,9 +197,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2 h1:LR89qFljJ48s990kEKGsk213yIJDPI4205OKOzbURK8= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -236,14 +209,11 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7 h1:ux/56T2xqZO/3cP1I2F86qpeoYPCOzk+KF/UH/Ar+lk= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -256,18 +226,15 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -303,15 +270,12 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -327,7 +291,6 @@ github.com/spf13/afero v1.3.4 h1:8q6vk3hthlpb2SouZcnBVKboxWQWMDNF38bwholZrJc= github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -336,6 +299,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= @@ -348,7 +312,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -364,7 +327,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -398,10 +360,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -417,7 +377,6 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -537,7 +496,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -549,7 +507,6 @@ golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -711,9 +668,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -733,11 +688,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/adapters/grpc/errors.go b/pkg/adapters/grpc/errors.go index 46c72bab7..d7d6605d0 100644 --- a/pkg/adapters/grpc/errors.go +++ b/pkg/adapters/grpc/errors.go @@ -15,25 +15,127 @@ package grpc import ( + "fmt" + "reflect" + + v1 "github.com/nitric-dev/membrane/interfaces/nitric/v1" "github.com/nitric-dev/membrane/pkg/plugins/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" ) // Provides GRPC error reporting func NewGrpcError(operation string, err error) error { if pe, ok := err.(*errors.PluginError); ok { - return newGrpcErrorWithCode(codes.Code(errors.Code(pe)), operation, pe) + code := codes.Code(errors.Code(pe)) + + ed := &v1.ErrorDetails{} + ed.Message = pe.Msg + if pe.Cause != nil { + detailsMap := map[string]interface{}{ + "cause": pe.Cause.Error(), + } + if detailsStruct, err := structpb.NewStruct(detailsMap); err == nil { + ed.Details = detailsStruct + } + } + ed.Scope = &v1.ErrorScope{ + Service: operation, + Plugin: pe.Plugin, + } + if len(pe.Args) > 0 { + args := make(map[string]string) + for k, v := range pe.Args { + args[k] = LogArg(v) + } + ed.Scope.Args = args + } + + s := status.New(code, pe.Msg) + s, _ = s.WithDetails(ed) + + return s.Err() + } else { return newGrpcErrorWithCode(codes.Internal, operation, err) } } func newGrpcErrorWithCode(code codes.Code, operation string, err error) error { - return status.Errorf(code, "%s: %v", operation, err) + ed := &v1.ErrorDetails{} + ed.Message = err.Error() + ed.Scope = &v1.ErrorScope{ + Service: operation, + } + + s := status.New(code, err.Error()) + s, _ = s.WithDetails(ed) + + return s.Err() } // Provides generic error for unregistered plugins func NewPluginNotRegisteredError(plugin string) error { - return status.Errorf(codes.Unimplemented, "%s plugin not registered", plugin) + ed := &v1.ErrorDetails{} + ed.Message = fmt.Sprintf("%s plugin not registered", plugin) + + s := status.New(codes.Unimplemented, ed.Message) + s, _ = s.WithDetails(ed) + + return s.Err() +} + +func LogArg(arg interface{}) string { + value := getValue(arg) + + if value.Kind() == reflect.Struct { + + str := "{" + for i := 0; i < value.NumField(); i++ { + + fieldType := value.Type().Field(i) + tag := fieldType.Tag.Get("log") + if tag == "" || tag == "-" { + continue + } + + if len(str) > 1 { + str += ", " + } + + field := value.Field(i) + str += fieldType.Name + ": " + LogArg(field.Interface()) + } + str += "}" + + return str + + } else if value.Kind() == reflect.Map { + str := "{" + + for k, v := range arg.(map[string]interface{}) { + if len(str) > 1 { + str += ", " + } + str += fmt.Sprintf("%v", k) + ": " + LogArg(v) + } + + str += "}" + + return str + + } else { + return fmt.Sprintf("%v", arg) + } +} + +func getValue(x interface{}) reflect.Value { + val := reflect.ValueOf(x) + + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + + return val } diff --git a/pkg/adapters/grpc/errors_test.go b/pkg/adapters/grpc/errors_test.go index 9497249f8..4b185ac9b 100644 --- a/pkg/adapters/grpc/errors_test.go +++ b/pkg/adapters/grpc/errors_test.go @@ -24,25 +24,50 @@ import ( . "github.com/onsi/gomega" ) +type SecretValue struct { + Type string `log:"Type"` + Factor int `log:"-"` + Value string +} + +type Secret struct { + Name string `json:"Name" log:"Name"` + Version string `json:"Version" log:"Version"` + Value *SecretValue `json:"Value" log:"Value"` +} + var _ = Describe("GRPC Errors", func() { Context("GrpcError", func() { When("plugin.errors.InvalidArgument", func() { It("Should report GRPC IllegalArgument error", func() { - newErr := errors.ErrorsWithScope("test") + newErr := errors.ErrorsWithScope("test", nil) + err := newErr( + codes.InvalidArgument, + "bad param", + nil, + ) + grpcErr := grpc.NewGrpcError("BadServer.BadCall", err) + Expect(grpcErr.Error()).To(ContainSubstring("rpc error: code = InvalidArgument desc = bad param")) + }) + }) + When("plugin.errors.InvalidArgument args", func() { + It("Should report GRPC IllegalArgument error with args", func() { + args := map[string]interface{}{"key": "value"} + newErr := errors.ErrorsWithScope("test", args) err := newErr( codes.InvalidArgument, "bad param", nil, ) grpcErr := grpc.NewGrpcError("BadServer.BadCall", err) - Expect(grpcErr.Error()).To(ContainSubstring("rpc error: code = InvalidArgument desc = BadServer.BadCall: test([]): bad param")) + Expect(grpcErr.Error()).To(ContainSubstring("rpc error: code = InvalidArgument desc = bad param")) }) }) When("Standard Error", func() { It("Should report GRPC Internal error", func() { err := fmt.Errorf("internal error") err = grpc.NewGrpcError("BadServer.BadCall", err) - Expect(err.Error()).To(ContainSubstring("rpc error: code = Internal desc = BadServer.BadCall: internal error")) + Expect(err.Error()).To(ContainSubstring("rpc error: code = Internal desc = internal error")) }) }) }) @@ -55,4 +80,69 @@ var _ = Describe("GRPC Errors", func() { }) }) }) + + Context("Logging Arg", func() { + When("string", func() { + It("return string value", func() { + Expect(grpc.LogArg("string")).To(BeEquivalentTo("string")) + }) + }) + + When("int", func() { + It("return string value", func() { + Expect(grpc.LogArg(123)).To(BeEquivalentTo("123")) + }) + }) + + When("bool", func() { + It("return string value", func() { + Expect(grpc.LogArg(true)).To(BeEquivalentTo("true")) + }) + }) + + When("float", func() { + It("return string value", func() { + Expect(grpc.LogArg(3.1415)).To(BeEquivalentTo("3.1415")) + }) + }) + + When("struct", func() { + It("return string value", func() { + + data := Secret{ + Name: "name", + Version: "3", + Value: &SecretValue{ + Type: "key", + Factor: 2, + Value: "2a4wijgPq0PpwJ76IjT7&lTBZ$5SGRcq", + }, + } + + value := grpc.LogArg(data) + Expect(value).To(BeEquivalentTo("{Name: name, Version: 3, Value: {Type: key}}")) + }) + }) + + When("map", func() { + It("return string value", func() { + secret := Secret{ + Name: "name", + Version: "3", + Value: &SecretValue{ + Type: "key", + Factor: 2, + Value: "2a4wijgPq0PpwJ76IjT7&lTBZ$5SGRcq", + }, + } + + valueMap := map[string]interface{}{ + "key": "value", + "secret": secret, + } + value := grpc.LogArg(valueMap) + Expect(value).To(BeEquivalentTo("{key: value, secret: {Name: name, Version: 3, Value: {Type: key}}}")) + }) + }) + }) }) diff --git a/pkg/adapters/grpc/event_grpc_test.go b/pkg/adapters/grpc/event_grpc_test.go index 9cb3d4998..83f0ae400 100644 --- a/pkg/adapters/grpc/event_grpc_test.go +++ b/pkg/adapters/grpc/event_grpc_test.go @@ -16,7 +16,6 @@ package grpc_test import ( "context" - "fmt" "github.com/nitric-dev/membrane/pkg/adapters/grpc" @@ -36,7 +35,6 @@ type MockEventService struct { } func (m *MockEventService) Publish(topic string, event *events.NitricEvent) error { - fmt.Printf("Publish called %v", event) m.PublishTopic = topic m.PublishEvent = event return m.PublishError diff --git a/pkg/plugins/document/boltdb/boltdb.go b/pkg/plugins/document/boltdb/boltdb.go index 26554516f..3dce599de 100644 --- a/pkg/plugins/document/boltdb/boltdb.go +++ b/pkg/plugins/document/boltdb/boltdb.go @@ -59,7 +59,9 @@ func (d BoltDoc) String() string { func (s *BoltDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "BoltDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -105,7 +107,9 @@ func (s *BoltDocService) Get(key *document.Key) (*document.Document, error) { func (s *BoltDocService) Set(key *document.Key, content map[string]interface{}) error { newErr := errors.ErrorsWithScope( "BoltDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -151,7 +155,9 @@ func (s *BoltDocService) Set(key *document.Key, content map[string]interface{}) func (s *BoltDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "BoltDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -212,7 +218,9 @@ func (s *BoltDocService) Delete(key *document.Key) error { func (s *BoltDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "BoltDocService.Query", - fmt.Sprintf("collection=%v", collection), + map[string]interface{}{ + "collection": collection, + }, ) if err := document.ValidateQueryCollection(collection); err != nil { diff --git a/pkg/plugins/document/dynamodb/dynamodb.go b/pkg/plugins/document/dynamodb/dynamodb.go index 31db0205e..068728a37 100644 --- a/pkg/plugins/document/dynamodb/dynamodb.go +++ b/pkg/plugins/document/dynamodb/dynamodb.go @@ -47,7 +47,9 @@ type DynamoDocService struct { func (s *DynamoDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "DynamoDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) err := document.ValidateKey(key) @@ -119,7 +121,9 @@ func (s *DynamoDocService) Get(key *document.Key) (*document.Document, error) { func (s *DynamoDocService) Set(key *document.Key, value map[string]interface{}) error { newErr := errors.ErrorsWithScope( "DynamoDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -175,7 +179,9 @@ func (s *DynamoDocService) Set(key *document.Key, value map[string]interface{}) func (s *DynamoDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "DynamoDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -258,7 +264,9 @@ func (s *DynamoDocService) Delete(key *document.Key) error { func (s *DynamoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "DynamoDocService.Query", - fmt.Sprintf("collection=%v", collection), + map[string]interface{}{ + "collection": collection, + }, ) if err := document.ValidateQueryCollection(collection); err != nil { diff --git a/pkg/plugins/document/firestore/firestore.go b/pkg/plugins/document/firestore/firestore.go index cb8c47818..e38d06a27 100644 --- a/pkg/plugins/document/firestore/firestore.go +++ b/pkg/plugins/document/firestore/firestore.go @@ -43,7 +43,9 @@ type FirestoreDocService struct { func (s *FirestoreDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "FirestoreDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -79,7 +81,9 @@ func (s *FirestoreDocService) Get(key *document.Key) (*document.Document, error) func (s *FirestoreDocService) Set(key *document.Key, value map[string]interface{}) error { newErr := errors.ErrorsWithScope( "FirestoreDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -114,7 +118,9 @@ func (s *FirestoreDocService) Set(key *document.Key, value map[string]interface{ func (s *FirestoreDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "FirestoreDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -182,7 +188,9 @@ func (s *FirestoreDocService) Delete(key *document.Key) error { func (s *FirestoreDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "FirestoreDocService.Query", - fmt.Sprintf("collection=%v", collection), + map[string]interface{}{ + "collection": collection, + }, ) if err := document.ValidateQueryCollection(collection); err != nil { diff --git a/pkg/plugins/document/mongodb/mongodb.go b/pkg/plugins/document/mongodb/mongodb.go index 3e99251b0..5211b0412 100644 --- a/pkg/plugins/document/mongodb/mongodb.go +++ b/pkg/plugins/document/mongodb/mongodb.go @@ -37,8 +37,8 @@ const ( mongoDBSetDirectEnvVarName = "MONGODB_DIRECT" primaryKeyAttr = "_id" - parentKeyAttr = "_parent_id" - childrenAttr = "_child_colls" + parentKeyAttr = "_parent_id" + childrenAttr = "_child_colls" ) // Mapping to mongo operators, startsWith will be handled within the function @@ -51,11 +51,9 @@ var mongoOperatorMap = map[string]string{ ">": "$gt", } - - type MongoDocService struct { client *mongo.Client - db *mongo.Database + db *mongo.Database context context.Context document.UnimplementedDocumentPlugin } @@ -63,7 +61,9 @@ type MongoDocService struct { func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "MongoDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -77,7 +77,7 @@ func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { col := s.getCollection(key) docRef := bson.M{primaryKeyAttr: key.Id} - var value map[string]interface{}; + var value map[string]interface{} opts := options.FindOne() @@ -85,7 +85,7 @@ func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { opts.SetProjection(bson.M{primaryKeyAttr: 0, parentKeyAttr: 0, childrenAttr: 0}) err := col.FindOne(s.context, docRef, opts).Decode(&value) - + if err != nil { var code = codes.Internal if err == mongo.ErrNoDocuments { @@ -112,7 +112,9 @@ func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) error { newErr := errors.ErrorsWithScope( "MongoDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -131,20 +133,18 @@ func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) e ) } - - coll := s.getCollection(key) value = mapKeys(key, value) - + opts := options.Update().SetUpsert(true) - + filter := bson.M{primaryKeyAttr: key.Id} update := bson.D{{"$set", value}} - - _, err := coll.UpdateOne(s.context, filter, update, opts); - + + _, err := coll.UpdateOne(s.context, filter, update, opts) + if err != nil { return newErr( codes.Internal, @@ -156,7 +156,7 @@ func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) e // add references if key.Collection.Parent != nil { err := s.updateChildReferences(key, coll.Name(), "$addToSet") - + if err != nil { return newErr( codes.Internal, @@ -172,7 +172,9 @@ func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) e func (s *MongoDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "MongoDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -187,8 +189,8 @@ func (s *MongoDocService) Delete(key *document.Key) error { filter := bson.M{primaryKeyAttr: key.Id} - opts := options.FindOneAndDelete().SetProjection(bson.M{ childrenAttr: 1, primaryKeyAttr: 0 }) - + opts := options.FindOneAndDelete().SetProjection(bson.M{childrenAttr: 1, primaryKeyAttr: 0}) + var deletedDocument map[string]interface{} // Delete document @@ -222,7 +224,7 @@ func (s *MongoDocService) Delete(key *document.Key) error { // clean references if none left if key.Collection.Parent != nil { err := s.updateChildReferences(key, coll.Name(), "$pull") - + if err != nil { return newErr( codes.Internal, @@ -231,14 +233,16 @@ func (s *MongoDocService) Delete(key *document.Key) error { ) } } - - return nil; + + return nil } func (s *MongoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "MongoDocService.Query", - fmt.Sprintf("collection=%v", collection), + map[string]interface{}{ + "collection": collection, + }, ) if err := document.ValidateQueryCollection(collection); err != nil { @@ -266,7 +270,7 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d opts := options.Find() opts.SetProjection(bson.M{childrenAttr: 0}) - + if limit > 0 { opts.SetLimit(int64(limit)) @@ -278,14 +282,12 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d for _, v := range strings.Split(tokens, "|") { vals = append(vals, v) } - + query[primaryKeyAttr] = bson.D{{"$gt", vals[len(vals)-1]}} } } } - - if collection.Parent != nil && collection.Parent.Id != "" { query[parentKeyAttr] = collection.Parent.Id } @@ -295,12 +297,12 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d if exp.Operator == "startsWith" { expVal := fmt.Sprintf("%v", exp.Value) endRangeValue := document.GetEndRangeValue(expVal) - + startsWith := bson.D{ {s.getOperator(">="), expVal}, {s.getOperator("<"), endRangeValue}, } - + query[expOperand] = startsWith } else { @@ -314,7 +316,7 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d orderByAttrib = expOperand } } - + queryResult := &document.QueryResult{ Documents: make([]document.Document, 0), } @@ -331,7 +333,7 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d defer cursor.Close(s.context) for cursor.Next(s.context) { - var docSnap map[string]interface{}; + var docSnap map[string]interface{} err := cursor.Decode(&docSnap) @@ -352,7 +354,7 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d Content: docSnap, Key: &document.Key{ Collection: collection, - Id: id, + Id: id, }, } @@ -369,7 +371,7 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d delete(docSnap, parentKeyAttr) } - + queryResult.Documents = append(queryResult.Documents, sdkDoc) // If query limit configured determine continue tokens @@ -396,7 +398,6 @@ func New() (document.DocumentService, error) { return nil, fmt.Errorf("MongoDB missing environment variable: %v", mongoDBConnectionStringEnvVarName) } - database := utils.GetEnv(mongoDBDatabaseEnvVarName, "") if database == "" { @@ -415,7 +416,7 @@ func New() (document.DocumentService, error) { if clientError != nil { return nil, fmt.Errorf("mongodb error creating client: %v", clientError) } - + connectError := client.Connect(ctx) if connectError != nil { @@ -426,7 +427,7 @@ func New() (document.DocumentService, error) { return &MongoDocService{ client: client, - db: db, + db: db, context: context.Background(), }, nil } @@ -436,7 +437,7 @@ func NewWithClient(client *mongo.Client, database string, ctx context.Context) d return &MongoDocService{ client: client, - db: db, + db: db, context: ctx, } } @@ -449,7 +450,7 @@ func mapKeys(key *document.Key, source map[string]interface{}) map[string]interf newMap[key] = value } - parentKey := key.Collection.Parent + parentKey := key.Collection.Parent newMap[primaryKeyAttr] = key.Id @@ -463,11 +464,11 @@ func mapKeys(key *document.Key, source map[string]interface{}) map[string]interf func (s *MongoDocService) updateChildReferences(key *document.Key, subCollectionName string, action string) error { parentColl := s.getCollection(key.Collection.Parent) filter := bson.M{primaryKeyAttr: key.Collection.Parent.Id} - referenceMeta := bson.M{ childrenAttr: subCollectionName } + referenceMeta := bson.M{childrenAttr: subCollectionName} update := bson.D{{action, referenceMeta}} opts := options.Update().SetUpsert(true) - _, err := parentColl.UpdateOne(s.context, filter, update, opts); + _, err := parentColl.UpdateOne(s.context, filter, update, opts) if err != nil { return err @@ -476,7 +477,7 @@ func (s *MongoDocService) updateChildReferences(key *document.Key, subCollection return nil } -func (s *MongoDocService) getCollection(key *document.Key) *mongo.Collection { +func (s *MongoDocService) getCollection(key *document.Key) *mongo.Collection { collectionNames := []string{} parentKey := key.Collection.Parent @@ -492,4 +493,4 @@ func (s *MongoDocService) getCollection(key *document.Key) *mongo.Collection { func (s *MongoDocService) getOperator(operator string) string { return mongoOperatorMap[operator] -} \ No newline at end of file +} diff --git a/pkg/plugins/document/plugin.go b/pkg/plugins/document/plugin.go index 00025d38a..3aa00888e 100644 --- a/pkg/plugins/document/plugin.go +++ b/pkg/plugins/document/plugin.go @@ -23,17 +23,13 @@ import "fmt" const MaxSubCollectionDepth int = 1 type Collection struct { - Name string - Parent *Key + Name string `log:"Name"` + Parent *Key `log:"Parent"` } type Key struct { - Collection *Collection - Id string -} - -func (k *Key) String() string { - return fmt.Sprintf("Key{Collection: %v Id: %v}\n", k.Collection, k.Id) + Collection *Collection `log:"Collection"` + Id string `log:"Id"` } type Document struct { @@ -41,10 +37,6 @@ type Document struct { Content map[string]interface{} } -func (d *Document) String() string { - return fmt.Sprintf("Document{Content: %v}\n", d.Content) -} - type QueryExpression struct { Operand string Operator string diff --git a/pkg/plugins/errors/plugin_error.go b/pkg/plugins/errors/plugin_error.go index dfb55cfe1..5cb5ace42 100644 --- a/pkg/plugins/errors/plugin_error.go +++ b/pkg/plugins/errors/plugin_error.go @@ -21,59 +21,44 @@ import ( ) type PluginError struct { - code codes.Code - msg string - cause error + Code codes.Code + Msg string + Cause error + Plugin string + Args map[string]interface{} } func (p *PluginError) Unwrap() error { - return p.cause + return p.Cause } func (p *PluginError) Error() string { - if p.cause != nil { + if p.Cause != nil { // If the wrapped error is an ApiError than these should unwrap - return fmt.Sprintf("%s: \n %s", p.msg, p.cause.Error()) + return fmt.Sprintf("%s: \n %s", p.Msg, p.Cause.Error()) } - return fmt.Sprintf("%s", p.msg) + return fmt.Sprintf("%s", p.Msg) } // Code - returns a nitric api error code from an error or Unknown if the error was not a nitric api error func Code(e error) codes.Code { if pe, ok := e.(*PluginError); ok { - return pe.code + return pe.Code } return codes.Unknown } -// New - Creates a new nitric API error -func new(c codes.Code, msg string) error { - return &PluginError{ - code: c, - msg: msg, - } -} - -// NewWithCause - Creates a new nitric API error with the given error as it's cause -func newWithCause(c codes.Code, msg string, cause error) error { - return &PluginError{ - code: c, - msg: msg, - cause: cause, - } -} - // ErrorsWithScope - Returns a new reusable error factory with the given scope -func ErrorsWithScope(s string, ctx ...interface{}) func(c codes.Code, msg string, cause error) error { - return func(c codes.Code, msg string, cause error) error { - sMsg := fmt.Sprintf("%s(%v): %s", s, ctx, msg) - - if cause == nil { - return new(c, sMsg) +func ErrorsWithScope(scope string, args map[string]interface{}) func(c codes.Code, msg string, cause error) error { + return func(code codes.Code, msg string, cause error) error { + return &PluginError{ + Code: code, + Msg: msg, + Cause: cause, + Plugin: scope, + Args: args, } - - return newWithCause(c, sMsg, cause) } } diff --git a/pkg/plugins/events/dev/eventing.go b/pkg/plugins/events/dev/eventing.go index d80c662fb..d0c36dfb1 100644 --- a/pkg/plugins/events/dev/eventing.go +++ b/pkg/plugins/events/dev/eventing.go @@ -45,7 +45,10 @@ type LocalHttpeventsClient interface { func (s *LocalEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope( "LocalEventService.Publish", - fmt.Sprintf("topic=%s", topic), + map[string]interface{}{ + "topic": topic, + "event": event, + }, ) requestId := event.ID @@ -96,7 +99,7 @@ func (s *LocalEventService) Publish(topic string, event *events.NitricEvent) err } else { return newErr( codes.NotFound, - fmt.Sprintf("unable to find subscriber for topic"), + "unable to find subscriber for topic", nil, ) } diff --git a/pkg/plugins/events/event.go b/pkg/plugins/events/event.go index 33656a7f9..609c53ca8 100644 --- a/pkg/plugins/events/event.go +++ b/pkg/plugins/events/event.go @@ -15,7 +15,7 @@ package events // NitricEvent - An event for asynchronous processing and reactive programming type NitricEvent struct { - ID string `json:"id,omitempty"` - PayloadType string `json:"payloadType,omitempty"` + ID string `json:"id,omitempty" log:"ID"` + PayloadType string `json:"payloadType,omitempty" log:"PayloadType"` Payload map[string]interface{} `json:"payload,omitempty"` } diff --git a/pkg/plugins/events/pubsub/pubsub.go b/pkg/plugins/events/pubsub/pubsub.go index 2eddae8f7..5b828fb21 100644 --- a/pkg/plugins/events/pubsub/pubsub.go +++ b/pkg/plugins/events/pubsub/pubsub.go @@ -35,7 +35,7 @@ type PubsubEventService struct { } func (s *PubsubEventService) ListTopics() ([]string, error) { - newErr := errors.ErrorsWithScope("PubsubEventService.ListTopics") + newErr := errors.ErrorsWithScope("PubsubEventService.ListTopics", nil) iter := s.client.Topics(context.TODO()) var topics []string @@ -57,7 +57,10 @@ func (s *PubsubEventService) ListTopics() ([]string, error) { func (s *PubsubEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope( "PubsubEventService.Publish", - fmt.Sprintf("topic=%s", topic), + map[string]interface{}{ + "topic": topic, + "event": event, + }, ) ctx := context.TODO() diff --git a/pkg/plugins/events/sns/sns.go b/pkg/plugins/events/sns/sns.go index febbc706b..f731c9dda 100644 --- a/pkg/plugins/events/sns/sns.go +++ b/pkg/plugins/events/sns/sns.go @@ -56,7 +56,10 @@ func (s *SnsEventService) getTopicArnFromName(name *string) (*string, error) { func (s *SnsEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope( "SnsEventService.Publish", - fmt.Sprintf("topic=%s", topic), + map[string]interface{}{ + "topic": topic, + "event": event, + }, ) data, err := json.Marshal(event) @@ -103,7 +106,7 @@ func (s *SnsEventService) Publish(topic string, event *events.NitricEvent) error } func (s *SnsEventService) ListTopics() ([]string, error) { - newErr := errors.ErrorsWithScope("SnsEventService.ListTopics") + newErr := errors.ErrorsWithScope("SnsEventService.ListTopics", nil) topicsOutput, err := s.client.ListTopics(&sns.ListTopicsInput{}) diff --git a/pkg/plugins/queue/dev/queue.go b/pkg/plugins/queue/dev/queue.go index e338417c7..c9e66b6c0 100644 --- a/pkg/plugins/queue/dev/queue.go +++ b/pkg/plugins/queue/dev/queue.go @@ -47,7 +47,10 @@ type Item struct { func (s *DevQueueService) Send(queue string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( "DevQueueService.Send", - fmt.Sprintf("queue=%s", queue), + map[string]interface{}{ + "queue": queue, + "task": task, + }, ) if queue == "" { @@ -96,7 +99,10 @@ func (s *DevQueueService) Send(queue string, task queue.NitricTask) error { func (s *DevQueueService) SendBatch(q string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { newErr := errors.ErrorsWithScope( "DevQueueService.SendBatch", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "tasks.len": len(tasks), + }, ) if q == "" { @@ -156,7 +162,9 @@ func (s *DevQueueService) SendBatch(q string, tasks []queue.NitricTask) (*queue. func (s *DevQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( "DevQueueService.Receive", - fmt.Sprintf("options=%v", options), + map[string]interface{}{ + "options": options, + }, ) if options.QueueName == "" { @@ -179,6 +187,13 @@ func (s *DevQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricT var items []Item err = db.All(&items, storm.Limit(int(*options.Depth))) + if err != nil { + return nil, newErr( + codes.Internal, + "error reading tasks", + err, + ) + } poppedTasks := make([]queue.NitricTask, 0) for _, item := range items { @@ -211,7 +226,10 @@ func (s *DevQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricT func (s *DevQueueService) Complete(queue string, leaseId string) error { newErr := errors.ErrorsWithScope( "DevQueueService.Complete", - fmt.Sprintf("queue=%s", queue), + map[string]interface{}{ + "queue": queue, + "leaseId": leaseId, + }, ) if queue == "" { diff --git a/pkg/plugins/queue/plugin.go b/pkg/plugins/queue/plugin.go index 8ee15bf39..b54f058a6 100644 --- a/pkg/plugins/queue/plugin.go +++ b/pkg/plugins/queue/plugin.go @@ -39,12 +39,12 @@ type ReceiveOptions struct { // Nitric name for the queue. // // queueName is a required field - QueueName string `type:"string" required:"true"` + QueueName string `type:"string" required:"true" log:"QueueName"` // Max depth of queue messages to receive from the queue. // // If nil or 0, defaults to depth 1. - Depth *uint32 `type:"int" required:"false"` + Depth *uint32 `type:"int" required:"false" log:"Depth"` } func (p *ReceiveOptions) Validate() error { diff --git a/pkg/plugins/queue/pubsub/pubsub.go b/pkg/plugins/queue/pubsub/pubsub.go index 81c9b37cd..65f56d684 100644 --- a/pkg/plugins/queue/pubsub/pubsub.go +++ b/pkg/plugins/queue/pubsub/pubsub.go @@ -49,7 +49,10 @@ func generateQueueSubscription(queue string) string { func (s *PubsubQueueService) Send(queue string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( "PubsubQueueService.Send", - fmt.Sprintf("queue=%s", queue), + map[string]interface{}{ + "queue": queue, + "task": task, + }, ) // We'll be using pubsub with pull subscribers to facilitate queue functionality ctx := context.TODO() @@ -58,7 +61,7 @@ func (s *PubsubQueueService) Send(queue string, task queue.NitricTask) error { if exists, err := topic.Exists(ctx); !exists || err != nil { return newErr( codes.NotFound, - fmt.Sprintf("queue not found"), + "queue not found", err, ) } @@ -91,7 +94,10 @@ func (s *PubsubQueueService) Send(queue string, task queue.NitricTask) error { func (s *PubsubQueueService) SendBatch(q string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { newErr := errors.ErrorsWithScope( "PubsubQueueService.SendBatch", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "tasks.len": len(tasks), + }, ) // We'll be using pubsub with pull subscribers to facilitate queue functionality @@ -177,7 +183,9 @@ func (s *PubsubQueueService) getQueueSubscription(q string) (ifaces_pubsub.Subsc func (s *PubsubQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( "PubsubQueueService.Receive", - fmt.Sprintf("options=%v", options), + map[string]interface{}{ + "options": options, + }, ) if err := options.Validate(); err != nil { @@ -259,7 +267,10 @@ func (s *PubsubQueueService) Receive(options queue.ReceiveOptions) ([]queue.Nitr func (s *PubsubQueueService) Complete(q string, leaseId string) error { newErr := errors.ErrorsWithScope( "PubsubQueueService.Complete", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "leaseId": leaseId, + }, ) ctx := context.Background() diff --git a/pkg/plugins/queue/sqs/sqs.go b/pkg/plugins/queue/sqs/sqs.go index 63b673e1a..fb8940013 100644 --- a/pkg/plugins/queue/sqs/sqs.go +++ b/pkg/plugins/queue/sqs/sqs.go @@ -56,7 +56,10 @@ func (s *SQSQueueService) getUrlForQueueName(queueName string) (*string, error) func (s *SQSQueueService) Send(queueName string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( "SQSQueueService.Send", - fmt.Sprintf("queue=%s", queueName), + map[string]interface{}{ + "queue": queueName, + "task": task, + }, ) tasks := []queue.NitricTask{task} @@ -73,7 +76,10 @@ func (s *SQSQueueService) Send(queueName string, task queue.NitricTask) error { func (s *SQSQueueService) SendBatch(queueName string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { newErr := errors.ErrorsWithScope( "SQSQueueService.SendBatch", - fmt.Sprintf("queue=%s", queueName), + map[string]interface{}{ + "queue": queueName, + "tasks.len": len(tasks), + }, ) if url, err := s.getUrlForQueueName(queueName); err == nil { @@ -137,7 +143,9 @@ func (s *SQSQueueService) SendBatch(queueName string, tasks []queue.NitricTask) func (s *SQSQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( "SQSQueueService.Receive", - fmt.Sprintf("options=%v", options), + map[string]interface{}{ + "options": options, + }, ) if err := options.Validate(); err != nil { @@ -205,7 +213,10 @@ func (s *SQSQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricT func (s *SQSQueueService) Complete(q string, leaseId string) error { newErr := errors.ErrorsWithScope( "SQSQueueService.Complete", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "leaseId": leaseId, + }, ) if url, err := s.getUrlForQueueName(q); err == nil { diff --git a/pkg/plugins/queue/task.go b/pkg/plugins/queue/task.go index bd1c53bb8..c27e44b70 100644 --- a/pkg/plugins/queue/task.go +++ b/pkg/plugins/queue/task.go @@ -22,8 +22,8 @@ type FailedTask struct { // NitricTask - A task for asynchronous processing type NitricTask struct { - ID string `json:"id,omitempty"` - LeaseID string `json:"leaseId,omitempty"` - PayloadType string `json:"payloadType,omitempty"` + ID string `json:"id,omitempty" log:"ID"` + LeaseID string `json:"leaseId,omitempty" log:"LeaseID"` + PayloadType string `json:"payloadType,omitempty" log:"PayLoadType"` Payload map[string]interface{} `json:"payload,omitempty"` } diff --git a/pkg/plugins/secret/dev/dev.go b/pkg/plugins/secret/dev/dev.go index cfb82ff98..754aa2cb0 100644 --- a/pkg/plugins/secret/dev/dev.go +++ b/pkg/plugins/secret/dev/dev.go @@ -42,7 +42,12 @@ func (s *DevSecretService) secretFileName(sec *secret.Secret, v string) string { } func (s *DevSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { - newErr := errors.ErrorsWithScope("DevSecretService.Put") + newErr := errors.ErrorsWithScope( + "DevSecretService.Put", + map[string]interface{}{ + "secret": sec, + }, + ) if sec == nil { return nil, newErr(codes.InvalidArgument, "provide non-empty secret", nil) @@ -94,7 +99,12 @@ func (s *DevSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPu } func (s *DevSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { - newErr := errors.ErrorsWithScope("DevSecretService.Access") + newErr := errors.ErrorsWithScope( + "DevSecretService.Access", + map[string]interface{}{ + "version": sv, + }, + ) if sv.Secret.Name == "" { return nil, newErr( diff --git a/pkg/plugins/secret/secret_manager/secret_manager.go b/pkg/plugins/secret/secret_manager/secret_manager.go index fe8738924..febe6c53b 100644 --- a/pkg/plugins/secret/secret_manager/secret_manager.go +++ b/pkg/plugins/secret/secret_manager/secret_manager.go @@ -135,7 +135,12 @@ func (s *secretManagerSecretService) ensureSecret(sec *secret.Secret) (*secretma // Put - Creates a new secret if one doesn't exist, or just adds a new secret version func (s *secretManagerSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { - newErr := errors.ErrorsWithScope("SecretManagerSecretService.Put") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Put", + map[string]interface{}{ + "secret": sec, + }, + ) if err := validateNewSecret(sec, val); err != nil { return nil, newErr( @@ -186,7 +191,12 @@ func (s *secretManagerSecretService) Put(sec *secret.Secret, val []byte) (*secre // Get - Retrieves a secret given a name and a version func (s *secretManagerSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { - newErr := errors.ErrorsWithScope("SecretManagerSecretService.Access") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Access", + map[string]interface{}{ + "version": sv, + }, + ) fullName, err := s.buildSecretVersionName(sv) diff --git a/pkg/plugins/secret/secret_suite_test.go b/pkg/plugins/secret/secret_suite_test.go index d2c01c74f..fffd63491 100644 --- a/pkg/plugins/secret/secret_suite_test.go +++ b/pkg/plugins/secret/secret_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package secret +package secret_test import ( "testing" diff --git a/pkg/plugins/secret/secret_test.go b/pkg/plugins/secret/secret_test.go index 510631e4b..7e93178a2 100644 --- a/pkg/plugins/secret/secret_test.go +++ b/pkg/plugins/secret/secret_test.go @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package secret +package secret_test import ( + "github.com/nitric-dev/membrane/pkg/plugins/secret" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Unimplemented Secret Plugin Tests", func() { - uisp := &UnimplementedSecretPlugin{} + uisp := &secret.UnimplementedSecretPlugin{} Context("Put", func() { When("Calling Put on UnimplementedSecretPlugin", func() { diff --git a/pkg/plugins/secret/secrets_manager/secrets_manager.go b/pkg/plugins/secret/secrets_manager/secrets_manager.go index 5745e74fd..bd45394e3 100644 --- a/pkg/plugins/secret/secrets_manager/secrets_manager.go +++ b/pkg/plugins/secret/secrets_manager/secrets_manager.go @@ -49,7 +49,12 @@ func (s *secretsManagerSecretService) validateNewSecret(sec *secret.Secret, val } func (s *secretsManagerSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { - newErr := errors.ErrorsWithScope("SecretsManagerSecretService.Put") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Put", + map[string]interface{}{ + "secret": sec, + }, + ) if err := s.validateNewSecret(sec, val); err != nil { return nil, newErr( @@ -136,7 +141,12 @@ func (s *secretsManagerSecretService) Put(sec *secret.Secret, val []byte) (*secr } func (s *secretsManagerSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { - newErr := errors.ErrorsWithScope("SecretsManagerSecretService.Access") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Access", + map[string]interface{}{ + "version": sv, + }, + ) if len(sv.Secret.Name) == 0 { return nil, newErr( diff --git a/pkg/plugins/secret/types.go b/pkg/plugins/secret/types.go index ac3f86c13..1d7814e63 100644 --- a/pkg/plugins/secret/types.go +++ b/pkg/plugins/secret/types.go @@ -16,16 +16,16 @@ package secret // Secret - Represents a container for secret versions type Secret struct { - Name string + Name string `log:"Name"` } // SecretVersion - A version of a secret type SecretVersion struct { - Secret *Secret + Secret *Secret `log:"Secret"` // Version - the specific secret version this represents // Specifying "latest" will always retrieve the latest version of the secret - Version string + Version string `log:"Version"` } // SecretAccessResponse - Return value for a secret access request diff --git a/pkg/plugins/storage/boltdb/storage.go b/pkg/plugins/storage/boltdb/storage.go index 7b4b4d63f..34bc22f70 100644 --- a/pkg/plugins/storage/boltdb/storage.go +++ b/pkg/plugins/storage/boltdb/storage.go @@ -15,7 +15,6 @@ package boltdb_storage_service import ( - "fmt" "os" "strings" "time" @@ -45,8 +44,11 @@ type Object struct { func (s *BoltStorageService) Write(bucket string, key string, object []byte) error { newErr := errors.ErrorsWithScope( "BoltStorageService.Write", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + "object.len": len(object), + }, ) if bucket == "" { @@ -109,8 +111,10 @@ func (s *BoltStorageService) Write(bucket string, key string, object []byte) err func (s *BoltStorageService) Read(bucket string, key string) ([]byte, error) { newErr := errors.ErrorsWithScope( "BoltStorageService.Read", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if bucket == "" { @@ -156,8 +160,10 @@ func (s *BoltStorageService) Read(bucket string, key string) ([]byte, error) { func (s *BoltStorageService) Delete(bucket string, key string) error { newErr := errors.ErrorsWithScope( "BoltStorageService.Delete", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if bucket == "" { diff --git a/pkg/plugins/storage/s3/s3.go b/pkg/plugins/storage/s3/s3.go index ca84dff0e..f98b1abbc 100644 --- a/pkg/plugins/storage/s3/s3.go +++ b/pkg/plugins/storage/s3/s3.go @@ -83,8 +83,10 @@ func (s *S3StorageService) getBucketByName(bucket string) (*s3.Bucket, error) { func (s *S3StorageService) Read(bucket string, key string) ([]byte, error) { newErr := errors.ErrorsWithScope( "S3StorageService.Read", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if b, err := s.getBucketByName(bucket); err == nil { @@ -117,8 +119,11 @@ func (s *S3StorageService) Read(bucket string, key string) ([]byte, error) { func (s *S3StorageService) Write(bucket string, key string, object []byte) error { newErr := errors.ErrorsWithScope( "S3StorageService.Write", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + "object.len": len(object), + }, ) if b, err := s.getBucketByName(bucket); err == nil { @@ -151,8 +156,10 @@ func (s *S3StorageService) Write(bucket string, key string, object []byte) error func (s *S3StorageService) Delete(bucket string, key string) error { newErr := errors.ErrorsWithScope( "S3StorageService.Delete", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if b, err := s.getBucketByName(bucket); err == nil { diff --git a/pkg/plugins/storage/storage/storage.go b/pkg/plugins/storage/storage/storage.go index 986a7792e..7390b3420 100644 --- a/pkg/plugins/storage/storage/storage.go +++ b/pkg/plugins/storage/storage/storage.go @@ -61,8 +61,10 @@ func (s *StorageStorageService) getBucketByName(bucket string) (ifaces_gcloud_st func (s *StorageStorageService) Read(bucket string, key string) ([]byte, error) { newErr := errors.ErrorsWithScope( "StorageStorageService.Read", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) bucketHandle, err := s.getBucketByName(bucket) @@ -102,8 +104,11 @@ func (s *StorageStorageService) Read(bucket string, key string) ([]byte, error) func (s *StorageStorageService) Write(bucket string, key string, object []byte) error { newErr := errors.ErrorsWithScope( "StorageStorageService.Write", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + "object.len": len(object), + }, ) bucketHandle, err := s.getBucketByName(bucket) @@ -143,8 +148,10 @@ func (s *StorageStorageService) Write(bucket string, key string, object []byte) func (s *StorageStorageService) Delete(bucket string, key string) error { newErr := errors.ErrorsWithScope( "StorageStorageService.Delete", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) bucketHandle, err := s.getBucketByName(bucket) From 70cbca5766d2df7a438a3f04304c1fd13abaefb5 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 14 Sep 2021 08:52:50 +1000 Subject: [PATCH 11/83] chore: Update sdk repo name. --- .github/workflows/generate-sdks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-sdks.yaml b/.github/workflows/generate-sdks.yaml index d81061a66..bd925181c 100644 --- a/.github/workflows/generate-sdks.yaml +++ b/.github/workflows/generate-sdks.yaml @@ -13,7 +13,7 @@ jobs: uses: peter-evans/repository-dispatch@v1 with: token: ${{ secrets.NITRIC_BOT_TOKEN }} - repository: nitrictech/base-sdk + repository: nitrictech/apis event-type: generate client-payload: > { From 205dd6572fbe4a8dadac88c3613e18dc15dcc013 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 14 Sep 2021 12:09:15 +1000 Subject: [PATCH 12/83] ci: calculate the rc number for each dev release. --- .github/workflows/release-rc.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-rc.yaml b/.github/workflows/release-rc.yaml index 4953d057b..533dfe261 100644 --- a/.github/workflows/release-rc.yaml +++ b/.github/workflows/release-rc.yaml @@ -15,6 +15,8 @@ jobs: upload_url: ${{ steps.create_release.outputs.upload_url }} steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Bump version and push tag id: tag_version uses: mathieudutour/github-tag-action@v5.5 @@ -25,9 +27,9 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} release_branches: main,develop - - name: Calculate SHORT_SHA + - name: Calculate RC number id: vars - run: echo "::set-output name=sha_short::$(echo ${GITHUB_SHA} | cut -c1-8)" + run: echo "::set-output name=rc_num::$(git rev-list --merges --count origin/develop...origin/main)" - name: Create a GitHub release id: create_release @@ -38,6 +40,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} with: prerelease: true - tag_name: ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.sha_short }} - release_name: Release ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.sha_short }} - body: ${{ steps.tag_version.outputs.changelog }}-rc.${{ steps.vars.outputs.sha_short }} \ No newline at end of file + tag_name: ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.rc_num }} + release_name: Release ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.rc_num }} + body: ${{ steps.tag_version.outputs.changelog }}-rc.${{ steps.vars.outputs.rc_num }} \ No newline at end of file From 8e70b0ad6a605507480a1564e62611d0b002d027 Mon Sep 17 00:00:00 2001 From: medgar Date: Tue, 14 Sep 2021 14:16:14 +1000 Subject: [PATCH 13/83] chore:review changes --- contracts/proto/error/v1/error.proto | 6 ++---- pkg/adapters/grpc/errors.go | 8 +------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/contracts/proto/error/v1/error.proto b/contracts/proto/error/v1/error.proto index f8abd9418..485abe431 100644 --- a/contracts/proto/error/v1/error.proto +++ b/contracts/proto/error/v1/error.proto @@ -1,8 +1,6 @@ syntax = "proto3"; package nitric.error.v1; -import "google/protobuf/struct.proto"; - // protoc plugin options for code generation option go_package = "nitric/v1;v1"; option java_package = "io.nitric.proto.error.v1"; @@ -26,8 +24,8 @@ message ErrorDetails { // The developer error message, explaining the error and ideally solution. string message = 1; - // The error details. - google.protobuf.Struct details = 2; + // The error root cause. + string cause = 2; // The scope of the error. ErrorScope scope = 3; diff --git a/pkg/adapters/grpc/errors.go b/pkg/adapters/grpc/errors.go index d7d6605d0..385184cf7 100644 --- a/pkg/adapters/grpc/errors.go +++ b/pkg/adapters/grpc/errors.go @@ -22,7 +22,6 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/structpb" ) // Provides GRPC error reporting @@ -33,12 +32,7 @@ func NewGrpcError(operation string, err error) error { ed := &v1.ErrorDetails{} ed.Message = pe.Msg if pe.Cause != nil { - detailsMap := map[string]interface{}{ - "cause": pe.Cause.Error(), - } - if detailsStruct, err := structpb.NewStruct(detailsMap); err == nil { - ed.Details = detailsStruct - } + ed.Cause = pe.Cause.Error() } ed.Scope = &v1.ErrorScope{ Service: operation, From dde5308113362d66378938fafeb4f142471c143f Mon Sep 17 00:00:00 2001 From: medgar Date: Tue, 14 Sep 2021 14:30:34 +1000 Subject: [PATCH 14/83] fix:unit test --- pkg/adapters/grpc/errors_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/adapters/grpc/errors_test.go b/pkg/adapters/grpc/errors_test.go index 4b185ac9b..ed4a705f9 100644 --- a/pkg/adapters/grpc/errors_test.go +++ b/pkg/adapters/grpc/errors_test.go @@ -141,7 +141,8 @@ var _ = Describe("GRPC Errors", func() { "secret": secret, } value := grpc.LogArg(valueMap) - Expect(value).To(BeEquivalentTo("{key: value, secret: {Name: name, Version: 3, Value: {Type: key}}}")) + Expect(value).To(ContainSubstring("secret: {Name: name, Version: 3, Value: {Type: key}}")) + Expect(value).To(ContainSubstring("key: value")) }) }) }) From bcef02eecc519473358c5d044233f01f6f7c55f2 Mon Sep 17 00:00:00 2001 From: medgar Date: Thu, 16 Sep 2021 09:04:41 +1000 Subject: [PATCH 15/83] chore:README updates --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c807466f4..0f6410a35 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,9 @@ The Membrane is at the heart of the solution. Nitric applications communicate wi Additional services on our roadmap include: - - RDS - - Configuration - - Logging - Authentication + - Email + - Logging > If you have any requests or suggestions let us know in the issues. @@ -77,7 +76,7 @@ provider as an alternative to the fixed set of plugins in the static membranes. ### Requirements - Git - - Golang + - Golang (1.16) - Make - Docker - Google Protocol Buffers Compiler @@ -96,7 +95,7 @@ Download the Google Protobuf Compiler (standalone binary called `protoc`) from h ### Run unit tests ```bash -make tests +make test ``` ### Run integration tests ```bash From 7ec68bc7c84e2dc20814bf35d29f990db996c67d Mon Sep 17 00:00:00 2001 From: Jye Cusch Date: Thu, 16 Sep 2021 14:42:19 +1000 Subject: [PATCH 16/83] feat(aws): add ability to generate presigned URLs for files in buckets --- contracts/proto/storage/v1/storage.proto | 29 +++++++- makefile | 4 +- pkg/adapters/grpc/storage_grpc.go | 34 ++++++++++ pkg/plugins/storage/plugin.go | 17 +++++ pkg/plugins/storage/s3/s3.go | 59 +++++++++++++++- pkg/plugins/storage/s3/s3_test.go | 85 +++++++++++++++++++++++- pkg/plugins/storage/storage/storage.go | 2 +- tests/mocks/s3/mock.go | 2 +- 8 files changed, 224 insertions(+), 8 deletions(-) diff --git a/contracts/proto/storage/v1/storage.proto b/contracts/proto/storage/v1/storage.proto index 2be8aeeed..099205466 100644 --- a/contracts/proto/storage/v1/storage.proto +++ b/contracts/proto/storage/v1/storage.proto @@ -19,6 +19,8 @@ service StorageService { rpc Write (StorageWriteRequest) returns (StorageWriteResponse); // Delete an item from a bucket rpc Delete (StorageDeleteRequest) returns (StorageDeleteResponse); + // Generate a pre-signed URL for direct operations on an item + rpc PreSignUrl (StoragePreSignUrlRequest) returns (StoragePreSignUrlResponse); } // Request to put (create/update) a storage item @@ -68,4 +70,29 @@ message StorageDeleteRequest { } // Result of deleting a storage item -message StorageDeleteResponse {} \ No newline at end of file +message StorageDeleteResponse {} + +// Request to generate a pre-signed URL for a file to perform a specific operation, such as read or write. +message StoragePreSignUrlRequest { + // Nitric name of the bucket to retrieve from + // this will be automatically resolved to the provider specific bucket identifier. + string bucket_name = 1; + // Key of item to generate the signed URL for. + // The URL and the token it contains will only be valid for operations on this resource specifically. + string key = 2; + // Operation + enum Operation { + READ = 0; + WRITE = 1; + } + Operation operation = 3; + // Expiry time in seconds for the token included in the signed URL. + // Time starts from when the access token is generated, not when this request is made. + // e.g. time.Now().Add(expiry * time.Second) on the server + uint32 expiry = 4; +} + +message StoragePreSignUrlResponse { + // The pre-signed url, restricted to the operation, resource and expiry time specified in the request. + string url = 1; +} \ No newline at end of file diff --git a/makefile b/makefile index 943ce0b2f..71a71f5ba 100644 --- a/makefile +++ b/makefile @@ -194,5 +194,7 @@ generate-mocks: @echo Generating Mock Clients @mkdir -p mocks/mock_secret_manager @mkdir -p mocks/secrets_manager + @mkdir -p mocks/s3 @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/mock_secret_manager/mock.go - @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go \ No newline at end of file + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go \ No newline at end of file diff --git a/pkg/adapters/grpc/storage_grpc.go b/pkg/adapters/grpc/storage_grpc.go index 4d3d818e3..c86e23a8a 100644 --- a/pkg/adapters/grpc/storage_grpc.go +++ b/pkg/adapters/grpc/storage_grpc.go @@ -16,6 +16,7 @@ package grpc import ( "context" + "fmt" pb "github.com/nitric-dev/membrane/interfaces/nitric/v1" "github.com/nitric-dev/membrane/pkg/plugins/storage" @@ -86,6 +87,39 @@ func (s *StorageServiceServer) Delete(ctx context.Context, req *pb.StorageDelete } } +func convertOperation(operation pb.StoragePreSignUrlRequest_Operation) (storage.Operation, error) { + if operation == pb.StoragePreSignUrlRequest_READ { + return storage.READ, nil + } else if operation == pb.StoragePreSignUrlRequest_WRITE { + return storage.WRITE, nil + } + return 0, fmt.Errorf("unknown storage operation, supported operations are READ and WRITE") +} + +func (s *StorageServiceServer) PreSignUrl(ctx context.Context, req *pb.StoragePreSignUrlRequest) (*pb.StoragePreSignUrlResponse, error) { + if err := s.checkPluginRegistered(); err != nil { + return nil, err + } + + if err := req.ValidateAll(); err != nil { + return nil, newGrpcErrorWithCode(codes.InvalidArgument, "StorageService.PreSignUrl", err) + } + + intendedOp, err := convertOperation(req.GetOperation()) + // For safety, don't set a default operation (like read). Only perform known operations + if err != nil { + return nil, newGrpcErrorWithCode(codes.InvalidArgument, "StorageService.PreSignUrl", err) + } + + if url, err := s.storagePlugin.PreSignUrl(req.GetBucketName(), req.GetKey(), intendedOp, req.GetExpiry()); err == nil { + return &pb.StoragePreSignUrlResponse{ + Url: url, + }, nil + } else { + return nil, NewGrpcError("StorageService.PreSignUrl", err) + } +} + func NewStorageServiceServer(storagePlugin storage.StorageService) pb.StorageServiceServer { return &StorageServiceServer{ storagePlugin: storagePlugin, diff --git a/pkg/plugins/storage/plugin.go b/pkg/plugins/storage/plugin.go index c792f9d68..b1198461c 100644 --- a/pkg/plugins/storage/plugin.go +++ b/pkg/plugins/storage/plugin.go @@ -16,10 +16,23 @@ package storage import "fmt" +type Operation int + +const ( + READ Operation = iota + WRITE = iota +) + +func (op Operation) String() string { + // The order of this array must match the iota order above. + return [2]string{"READ", "WRITE"}[op] +} + type StorageService interface { Read(bucket string, key string) ([]byte, error) Write(bucket string, key string, object []byte) error Delete(bucket string, key string) error + PreSignUrl(bucket string, key string, operation Operation, expiry uint32) (string, error) } type UnimplementedStoragePlugin struct{} @@ -37,3 +50,7 @@ func (*UnimplementedStoragePlugin) Write(bucket string, key string, object []byt func (*UnimplementedStoragePlugin) Delete(bucket string, key string) error { return fmt.Errorf("UNIMPLEMENTED") } + +func (*UnimplementedStoragePlugin) PreSignUrl(bucket string, key string, operation Operation, expiry uint32) (string, error) { + return "", fmt.Errorf("UNIMPLEMENTED") +} diff --git a/pkg/plugins/storage/s3/s3.go b/pkg/plugins/storage/s3/s3.go index f98b1abbc..7006af1cd 100644 --- a/pkg/plugins/storage/s3/s3.go +++ b/pkg/plugins/storage/s3/s3.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "net/http" + "time" "github.com/nitric-dev/membrane/pkg/utils" @@ -33,13 +34,13 @@ import ( ) const ( - // AWS API neglects to include a constant for this error code. + // ErrCodeNoSuchTagSet - AWS API neglects to include a constant for this error code. ErrCodeNoSuchTagSet = "NoSuchTagSet" ) // S3StorageService - Is the concrete implementation of AWS S3 for the Nitric Storage Plugin type S3StorageService struct { - storage.UnimplementedStoragePlugin + //storage.UnimplementedStoragePlugin client s3iface.S3API } @@ -185,6 +186,60 @@ func (s *S3StorageService) Delete(bucket string, key string) error { return nil } +// PreSignUrl - generates a signed URL which can be used to perform direct operations on a file +// useful for large file uploads/downloads so they can bypass application code and work directly with S3 +func (s *S3StorageService) PreSignUrl(bucket string, key string, operation storage.Operation, expiry uint32) (string, error) { + newErr := errors.ErrorsWithScope( + "S3StorageService.PreSignUrl", + map[string]interface{}{ + "bucket": bucket, + "key": key, + "operation": operation.String(), + }, + ) + + if b, err := s.getBucketByName(bucket); err == nil { + switch operation { + case storage.READ: + req, _ := s.client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: b.Name, + Key: aws.String(key), + }) + url, err := req.Presign(time.Duration(expiry) * time.Second) + if err != nil { + return "", newErr( + codes.Internal, + "failed to generate pre-signed READ URL", + err, + ) + } + return url, err + case storage.WRITE: + req, _ := s.client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: b.Name, + Key: aws.String(key), + }) + url, err := req.Presign(time.Duration(expiry) * time.Second) + if err != nil { + return "", newErr( + codes.Internal, + "failed to generate pre-signed WRITE URL", + err, + ) + } + return url, err + default: + return "", fmt.Errorf("requested operation not supported for pre-signed AWS S3 urls") + } + } else { + return "", newErr( + codes.NotFound, + "unable to locate bucket", + err, + ) + } +} + // New creates a new default S3 storage plugin func New() (storage.StorageService, error) { awsRegion := utils.GetEnv("AWS_REGION", "us-east-1") diff --git a/pkg/plugins/storage/s3/s3_test.go b/pkg/plugins/storage/s3/s3_test.go index 49872dac5..0d1f111f9 100644 --- a/pkg/plugins/storage/s3/s3_test.go +++ b/pkg/plugins/storage/s3/s3_test.go @@ -15,8 +15,17 @@ package s3_service_test import ( - "github.com/nitric-dev/membrane/pkg/plugins/storage/s3" - "github.com/nitric-dev/membrane/tests/mocks/s3" + "fmt" + "net/http" + "net/url" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/golang/mock/gomock" + mock_s3iface "github.com/nitric-dev/membrane/mocks/s3" + s3_service "github.com/nitric-dev/membrane/pkg/plugins/storage/s3" + mock_s3 "github.com/nitric-dev/membrane/tests/mocks/s3" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -131,4 +140,76 @@ var _ = Describe("S3", func() { }) }) }) + When("PreSignUrl", func() { + When("The bucket exists", func() { + // Set up a mock bucket, with a single item + storage := make(map[string]map[string][]byte) + storage["test-bucket"] = make(map[string][]byte) + crtl := gomock.NewController(GinkgoT()) + mockStorageClient := mock_s3iface.NewMockS3API(crtl) + storagePlugin, _ := s3_service.NewWithClient(mockStorageClient) + + When("A URL is requested for a known operation", func() { + It("Should successfully generate the URL", func() { + + By("Calling ListBuckets to map the bucket name") + mockStorageClient.EXPECT().ListBuckets(gomock.Any()).Times(1).Return(&s3.ListBucketsOutput{ + Buckets: []*s3.Bucket{{ + Name: aws.String("test-bucket-aaa111"), + }}, + }, nil) + + mockStorageClient.EXPECT().GetBucketTagging(gomock.Any()).Times(1).Return(&s3.GetBucketTaggingOutput{TagSet: []*s3.Tag{{ + Key: aws.String("x-nitric-name"), + Value: aws.String("test-bucket"), + }}}, nil) + + presign := 0 + mockStorageClient.EXPECT().PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String("test-bucket-aaa111"), // the real bucket name should be provided here, not the nitric name + Key: aws.String("test-key"), + }).Times(1).Return(&request.Request{ + Operation: &request.Operation{ + Name: "", + HTTPMethod: "", + HTTPPath: "", + Paginator: nil, + // Unfortunately, PutObjectRequest returns a struct, instead of an interface, + // so we can't really mock it. However, if this BeforePresignFn returns an error + // it currently prevents the rest of the presign call and returns a blank url string. + // this is good enough to perform basic testing. + BeforePresignFn: func(r *request.Request) error { + presign += 1 + return fmt.Errorf("test error") + }, + }, + HTTPRequest: &http.Request{Host: "", URL: &url.URL{ + Scheme: "", + Opaque: "", + User: nil, + Host: "aws.example.com", + Path: "", + RawPath: "", + ForceQuery: false, + RawQuery: "", + Fragment: "", + RawFragment: "", + }}, + }, nil) + + url, err := storagePlugin.PreSignUrl("test-bucket", "test-key", 1, uint32(60)) + By("Returning an error") + // We always get an error due to inability to replace the Request with a mock + Expect(err).Should(HaveOccurred()) + + By("Returning a blank url") + // always blank - it's the best we can do without a real mock. + Expect(url).To(Equal("")) + }) + }) + }) + When("The bucket doesn't exist", func() { + + }) + }) }) diff --git a/pkg/plugins/storage/storage/storage.go b/pkg/plugins/storage/storage/storage.go index 7390b3420..ca3c44a34 100644 --- a/pkg/plugins/storage/storage/storage.go +++ b/pkg/plugins/storage/storage/storage.go @@ -31,7 +31,7 @@ import ( ) type StorageStorageService struct { - //storage.UnimplementedStoragePlugin + plugin.UnimplementedStoragePlugin client ifaces_gcloud_storage.StorageClient projectID string } diff --git a/tests/mocks/s3/mock.go b/tests/mocks/s3/mock.go index 3d03a48eb..bbbce600e 100644 --- a/tests/mocks/s3/mock.go +++ b/tests/mocks/s3/mock.go @@ -30,7 +30,7 @@ type MockBucket struct { Tags map[string]string } -// MockS3Client - Provides and S3API complient mock interface +// MockS3Client - Provides an S3API compliant mock interface type MockS3Client struct { sync.RWMutex s3iface.S3API From 8178577d24834f17521b6f33950deace9ed7aea7 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 17 Aug 2021 14:21:33 +1000 Subject: [PATCH 17/83] feat: Add initial Key Vault plugin --- go.mod | 8 +- go.sum | 34 +- makefile | 6 +- pkg/plugins/secret/key_vault/key_vault.go | 152 +++++++++ .../secret/key_vault/key_vault_suite_test.go | 13 + .../secret/key_vault/key_vault_test.go | 300 ++++++++++++++++++ .../secrets_manager/secrets_manager_test.go | 2 +- pkg/providers/azure/README.md | 10 + 8 files changed, 516 insertions(+), 9 deletions(-) create mode 100644 pkg/plugins/secret/key_vault/key_vault.go create mode 100644 pkg/plugins/secret/key_vault/key_vault_suite_test.go create mode 100644 pkg/plugins/secret/key_vault/key_vault_test.go diff --git a/go.mod b/go.mod index 5994a9cda..9ce4f4b38 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,10 @@ require ( cloud.google.com/go/firestore v1.5.0 cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.10.0 - github.com/Azure/azure-sdk-for-go v51.3.0+incompatible + github.com/Azure/azure-sdk-for-go v56.2.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2 + github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1 github.com/Azure/go-autorest/autorest v0.11.18 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/DataDog/zstd v1.4.8 // indirect @@ -32,8 +35,9 @@ require ( go.etcd.io/bbolt v1.3.5 go.mongodb.org/mongo-driver v1.7.1 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 + golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.5 google.golang.org/api v0.40.0 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c diff --git a/go.sum b/go.sum index bf43d2d59..00a831f66 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,24 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v51.3.0+incompatible h1:Y3wR7C5Sj0nZG3VhkePF5hK7zNCS5yeImN/k2CWB+u8= -github.com/Azure/azure-sdk-for-go v51.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v55.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v56.2.0+incompatible h1:2GrG1JkTSMqLquy1pqVsjeRJhNtZLjss2+rx8ogZXx4= +github.com/Azure/azure-sdk-for-go v56.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0 h1:HQQoaSGOh32mpoRkpLDjkngMwYJqkxu93FRx0epdLHE= +github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0/go.mod h1:BSKvHb/5cy8j4hahIInXH92X/2zGJ3TxKF6b9pw1Btg= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.14.0/go.mod h1:pElNP+u99BvCZD+0jOlhI9OC/NB2IDTOTGZOZH0Qhq8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.16.2 h1:UC4vfOhW2l0f2QOCQpOxJS4/K6oKFy2tQZE+uWU1MEo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.16.2/go.mod h1:MVdrcUC4Hup35qHym3VdzoW+NBgBxrta9Vei97jRtM8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.1/go.mod h1:acANgl9stsT5xflESXKjZx4rhZJSr0TGgTDYY0xJPIE= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2 h1:3W8umQHRg0DXV5KvmbqU43uFi8MKvqLHQCE8L8v6Xds= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2/go.mod h1:acANgl9stsT5xflESXKjZx4rhZJSr0TGgTDYY0xJPIE= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.5.0/go.mod h1:k4KbFSunV/+0hOHL1vyFaPsiYQ1Vmvy1TBpmtvCDLZM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.5.1 h1:vx8McI56N5oLSQu8xa+xdiE0fjQq8W8Zt49vHP8Rygw= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.5.1/go.mod h1:k4KbFSunV/+0hOHL1vyFaPsiYQ1Vmvy1TBpmtvCDLZM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1 h1:GZP4qFuNKtt7Mq8PY2zCsgJOO0mqa6cyXZb0u1q+EH8= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1/go.mod h1:2TVZ9s3y3yhuyfHnId/SjaZ2Wly3EPcD7SvzYubFdrM= +github.com/Azure/azure-sdk-for-go/sdk/to v0.1.4 h1:3w4gk+uYOwplGhID1fDP305/8bI5Aug3URoC1V493KU= +github.com/Azure/azure-sdk-for-go/sdk/to v0.1.4/go.mod h1:UL/d4lvWAzSJUuX+19uKdN0ktyjoOyQhgY+HWNgtIYI= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= @@ -166,6 +182,10 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b h1:KwI0NOpYd3rzKojfjeRerF7rzjeTwvJARVsgGf5TWmY= +github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= +github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab h1:+qfOxKbnAqDNCoFUNHxudKs8Z14T5EBYntAeWIeI1eA= +github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= github.com/google/addlicense v1.0.0/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -349,6 +369,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -416,14 +437,16 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -493,6 +516,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210223095934-7937bea0104d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -503,8 +527,10 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/makefile b/makefile index 71a71f5ba..c5ddaec59 100644 --- a/makefile +++ b/makefile @@ -193,8 +193,10 @@ build-all-binaries: clean generate-proto generate-mocks: @echo Generating Mock Clients @mkdir -p mocks/mock_secret_manager - @mkdir -p mocks/secrets_manager + @mkdir -p mocks/mock_secrets_manager + @mkdir -p mocks/mock_key_vault @mkdir -p mocks/s3 @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/mock_secret_manager/mock.go - @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/mock_secrets_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/mock_key_vault/mock.go @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go \ No newline at end of file diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go new file mode 100644 index 000000000..b1eb2905f --- /dev/null +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -0,0 +1,152 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package key_vault_secret_service + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/armcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault" + "github.com/nitric-dev/membrane/pkg/plugins" + "github.com/nitric-dev/membrane/pkg/plugins/secret" + "github.com/nitric-dev/membrane/pkg/utils" +) + +const DEFAULT_SUBSCRIPTION_ID = "subscription-id" +const DEFAULT_RESOURCE_GROUP = "resource-group" +const DEFAULT_VAULT_NAME = "vault-name" + +// KeyVaultClient - iface that exposes utilized subset of generated KeyVaultSecretClient +// Used with gomock to assert create client -> service interaction in unit tests +type KeyVaultClient interface { + Get(ctx context.Context, resourceGroupName string, vaultName string, secretName string, options *armkeyvault.SecretsGetOptions) (armkeyvault.SecretResponse, error) + CreateOrUpdate(ctx context.Context, resourceGroupName string, vaultName string, secretName string, parameters armkeyvault.SecretCreateOrUpdateParameters, options *armkeyvault.SecretsCreateOrUpdateOptions) (armkeyvault.SecretResponse, error) +} +type keyVaultSecretService struct { + secret.UnimplementedSecretPlugin + client KeyVaultClient + resourceGroupName string + vaultName string +} + +func validateNewSecret(sec *secret.Secret, val []byte) error { + if sec == nil { + return plugins.NewInvalidArgError("provide non-nil secret") + } + if len(sec.Name) == 0 { + return plugins.NewInvalidArgError("provide non-blank secret name") + } + if len(val) == 0 { + return plugins.NewInvalidArgError("provide non-blank secret value") + } + + return nil +} +func validateSecretVersion(sec *secret.SecretVersion) error { + if sec == nil { + return plugins.NewInvalidArgError("provide non-nil versioned secret") + } + if sec.Secret == nil { + return plugins.NewInvalidArgError("provide non-nil secret") + } + if len(sec.Secret.Name) == 0 { + return plugins.NewInvalidArgError("provide non-blank secret name") + } + if len(sec.Version) == 0 { + return plugins.NewInvalidArgError("provide non-blank secret version") + } + return nil +} + +func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { + if err := validateNewSecret(sec, val); err != nil { + return nil, err + } + ctx := context.Background() + stringVal := string(val[:]) + result, err := s.client.CreateOrUpdate( + ctx, + s.resourceGroupName, + s.vaultName, + sec.Name, + armkeyvault.SecretCreateOrUpdateParameters{ + Properties: &armkeyvault.SecretProperties{ + Value: &stringVal, + }, + }, + nil, + ) + if err != nil { + return nil, fmt.Errorf("error putting secret %v", err) + } + return &secret.SecretPutResponse{ + SecretVersion: &secret.SecretVersion{ + Secret: &secret.Secret{ + Name: sec.Name, + }, + Version: *result.Secret.Properties.SecretURIWithVersion, + }, + }, nil +} + +//GET https://{vaultBaseUrl}/secrets/{secret-name}/{secret-version}?api-version={api-version} +func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { + if err := validateSecretVersion(sv); err != nil { + return nil, err + } + ctx := context.Background() + result, err := s.client.Get( + ctx, + s.resourceGroupName, + s.vaultName, + sv.Version, + &armkeyvault.SecretsGetOptions{}, + ) + if err != nil { + return nil, fmt.Errorf("error accessing secret %v", err) + } + return &secret.SecretAccessResponse{ + // Return the original secret version payload + SecretVersion: &secret.SecretVersion{ + Secret: &secret.Secret{ + Name: sv.Secret.Name, + }, + Version: *result.Secret.Properties.SecretURIWithVersion, + }, + Value: []byte(*result.Secret.Properties.Value), + }, nil +} + +// New - Creates a new Nitric secret service with Azure Key Vault Provider +func New() (secret.SecretService, error) { + subscriptionId := utils.GetEnv("AZURE_SUBSCRIPTION_ID", DEFAULT_SUBSCRIPTION_ID) + resouceGroup := utils.GetEnv("AZURE_RESOURCE_GROUP", DEFAULT_RESOURCE_GROUP) + vaultName := utils.GetEnv("AZURE_VAULT_NAME", DEFAULT_VAULT_NAME) + + credentials, credentialsError := azidentity.NewDefaultAzureCredential(nil) + if credentialsError != nil { + return nil, fmt.Errorf("azure credentials error: %v", credentialsError) + } + conn := armcore.NewDefaultConnection(credentials, nil) + client := armkeyvault.NewSecretsClient(conn, subscriptionId) + + return &keyVaultSecretService{ + client: client, + resourceGroupName: resouceGroup, + vaultName: vaultName, + }, nil +} diff --git a/pkg/plugins/secret/key_vault/key_vault_suite_test.go b/pkg/plugins/secret/key_vault/key_vault_suite_test.go new file mode 100644 index 000000000..ca237db57 --- /dev/null +++ b/pkg/plugins/secret/key_vault/key_vault_suite_test.go @@ -0,0 +1,13 @@ +package key_vault_secret_service + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestSecretManager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Key Vault Suite") +} diff --git a/pkg/plugins/secret/key_vault/key_vault_test.go b/pkg/plugins/secret/key_vault/key_vault_test.go new file mode 100644 index 000000000..f1b1774c5 --- /dev/null +++ b/pkg/plugins/secret/key_vault/key_vault_test.go @@ -0,0 +1,300 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package key_vault_secret_service + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault" + "github.com/golang/mock/gomock" + mocks "github.com/nitric-dev/membrane/mocks/mock_key_vault" + "github.com/nitric-dev/membrane/pkg/plugins/secret" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Key Vault", func() { + secretName := "secret-name" + secretVersion := "secret-name/version-name" + secretVal := []byte("Super Secret Message") + secretString := string(secretVal) + mockSecretResponse := armkeyvault.SecretResponse{ + Secret: &armkeyvault.Secret{ + Properties: &armkeyvault.SecretProperties{ + SecretURI: &secretName, + SecretURIWithVersion: &secretVersion, + Value: &secretString, + }, + }, + } + testSecret := &secret.Secret{ + Name: "secret-name", + } + testSecretVersion := &secret.SecretVersion{ + Secret: testSecret, + Version: secretVersion, + } + + When("Put", func() { + When("Given the Key Vault backend is available", func() { + When("Putting a Secret to an existing secret", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + + It("Should successfully store a secret", func() { + // Assert all methods are called at least their number of times + defer crtl.Finish() + + //Mocking expects + mockSecretClient.EXPECT().CreateOrUpdate( + context.Background(), + secretPlugin.resourceGroupName, + secretPlugin.vaultName, + testSecret.Name, + gomock.Any(), + gomock.Any(), + ).Return(mockSecretResponse, nil).Times(1) + + response, err := secretPlugin.Put(testSecret, secretVal) + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + By("Returning the service provided version id") + Expect(response.SecretVersion.Version).To(Equal(secretVersion)) + Expect(response.SecretVersion.Secret.Name).To(Equal(secretName)) + }) + }) + + When("Putting a Secret to a non-existent secret", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + It("Should successfully store a secret", func() { + defer crtl.Finish() + + //Mocking expects + mockSecretClient.EXPECT().CreateOrUpdate( + context.Background(), + secretPlugin.resourceGroupName, + secretPlugin.vaultName, + testSecret.Name, + gomock.Any(), + gomock.Any(), + ).Return(mockSecretResponse, nil).Times(1) + + response, err := secretPlugin.Put(testSecret, secretVal) + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + By("Returning the correct secret") + Expect(response.SecretVersion.Version).To(Equal(secretVersion)) + Expect(response.SecretVersion.Secret.Name).To(Equal(secretName)) + }) + }) + + When("Putting a nil secret", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + + It("Should invalidate the secret", func() { + _, err := secretPlugin.Put(nil, secretVal) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) + + When("Putting a secret with an empty name", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + + It("Should invalidate the secret", func() { + _, err := secretPlugin.Put(&secret.Secret{Name: ""}, secretVal) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) + + When("Putting a secret with an empty value", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + + It("Should invalidate the secret", func() { + _, err := secretPlugin.Put(testSecret, nil) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) + }) + }) + + When("Access", func() { + When("Given the Key Vault backend is available", func() { + When("The secret store exists", func() { + When("The secret exists", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + + It("Should successfully return a secret", func() { + defer crtl.Finish() + //Mocking expects + mockSecretClient.EXPECT().Get( + context.Background(), + secretPlugin.resourceGroupName, + secretPlugin.vaultName, + secretVersion, + gomock.Any(), + ).Return(mockSecretResponse, nil).Times(1) + + response, err := secretPlugin.Access(testSecretVersion) + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + By("Returning the correct secret") + Expect(response.Value).To(Equal(secretVal)) + Expect(response.SecretVersion).ToNot(BeNil()) + Expect(response.SecretVersion.Version).To(Equal(secretVersion)) + Expect(response.SecretVersion.Secret.Name).To(Equal(secretName)) + }) + }) + }) + When("The secret doesn't exist", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + It("Should return an error", func() { + defer crtl.Finish() + + mockSecretClient.EXPECT().Get( + context.Background(), + secretPlugin.resourceGroupName, + secretPlugin.vaultName, + secretVersion, + gomock.Any(), + ).Return(armkeyvault.SecretResponse{}, fmt.Errorf("secret does not exist")).Times(1) + + response, err := secretPlugin.Access(testSecretVersion) + By("returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(Equal("error accessing secret secret does not exist")) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty secret version is provided", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + It("Should return an error", func() { + defer crtl.Finish() + + response, err := secretPlugin.Access(nil) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty secret is provided", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + It("Should return an error", func() { + defer crtl.Finish() + + response, err := secretPlugin.Access(&secret.SecretVersion{Secret: nil, Version: secretVersion}) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty secret name is provided", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + It("Should return an error", func() { + defer crtl.Finish() + + response, err := secretPlugin.Access(&secret.SecretVersion{Secret: &secret.Secret{Name: ""}, Version: secretVersion}) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty version is provided", func() { + crtl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(crtl) + secretPlugin := &keyVaultSecretService{ + client: mockSecretClient, + resourceGroupName: "resource-group-name", + vaultName: "vault-name", + } + It("Should return an error", func() { + defer crtl.Finish() + + response, err := secretPlugin.Access(&secret.SecretVersion{Secret: testSecret, Version: ""}) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + }) + }) +}) diff --git a/pkg/plugins/secret/secrets_manager/secrets_manager_test.go b/pkg/plugins/secret/secrets_manager/secrets_manager_test.go index 91092d158..8cb748623 100644 --- a/pkg/plugins/secret/secrets_manager/secrets_manager_test.go +++ b/pkg/plugins/secret/secrets_manager/secrets_manager_test.go @@ -21,7 +21,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" secretsmanager "github.com/aws/aws-sdk-go/service/secretsmanager" "github.com/golang/mock/gomock" - mocks "github.com/nitric-dev/membrane/mocks/secrets_manager" + mocks "github.com/nitric-dev/membrane/mocks/mock_secrets_manager" "github.com/nitric-dev/membrane/pkg/plugins/secret" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/providers/azure/README.md b/pkg/providers/azure/README.md index d23f6f062..8ab4640ca 100644 --- a/pkg/providers/azure/README.md +++ b/pkg/providers/azure/README.md @@ -34,4 +34,14 @@ make azure-docker-debian > __Note:__ Separate distributions required between glibc/musl as dynamic linker is used for golang plugin support +### Credentials +AZURE_CLIENT_ID +AZURE_CLIENT_SECRET +AZURE_TENANT_ID + +AZURE_SUBSCRIPTION_ID +AZURE_RESOURCE_GROUP + +#### Key Vault +AZURE_VAULT_NAME From 9b2b58e317ebb5f4d6f379c540296befa6d08c73 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 17 Aug 2021 14:28:06 +1000 Subject: [PATCH 18/83] chore: Add missing license header --- .../secret/key_vault/key_vault_suite_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/plugins/secret/key_vault/key_vault_suite_test.go b/pkg/plugins/secret/key_vault/key_vault_suite_test.go index ca237db57..a8a45252a 100644 --- a/pkg/plugins/secret/key_vault/key_vault_suite_test.go +++ b/pkg/plugins/secret/key_vault/key_vault_suite_test.go @@ -1,3 +1,16 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package key_vault_secret_service import ( From 3466fd16b25c8c73f47deaedf50d6fa55803fa9c Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 17 Aug 2021 14:30:37 +1000 Subject: [PATCH 19/83] chore: Run go mod tidy --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 9ce4f4b38..12db76b0d 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/asdine/storm v2.1.2+incompatible github.com/aws/aws-lambda-go v1.20.0 github.com/aws/aws-sdk-go v1.36.12 - github.com/envoyproxy/protoc-gen-validate v0.6.1 // indirect + github.com/envoyproxy/protoc-gen-validate v0.6.1 github.com/golang/mock v1.4.4 github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.3 // indirect diff --git a/go.sum b/go.sum index 00a831f66..5523aa629 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b h1:KwI0NOpYd3rzKojfjeRerF7rzjeTwvJARVsgGf5TWmY= -github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab h1:+qfOxKbnAqDNCoFUNHxudKs8Z14T5EBYntAeWIeI1eA= github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= @@ -285,10 +283,11 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From 76e4c5341c122bb395749299bec6b51159561380 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 19 Aug 2021 11:56:32 +1000 Subject: [PATCH 20/83] feat: Add standard error codes --- pkg/plugins/secret/key_vault/key_vault.go | 54 +++++++++++++++++------ 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go index b1eb2905f..80aaf6636 100644 --- a/pkg/plugins/secret/key_vault/key_vault.go +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -21,7 +21,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/armcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault" - "github.com/nitric-dev/membrane/pkg/plugins" + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" "github.com/nitric-dev/membrane/pkg/plugins/secret" "github.com/nitric-dev/membrane/pkg/utils" ) @@ -45,36 +46,43 @@ type keyVaultSecretService struct { func validateNewSecret(sec *secret.Secret, val []byte) error { if sec == nil { - return plugins.NewInvalidArgError("provide non-nil secret") + return fmt.Errorf("provide non-nil secret") } if len(sec.Name) == 0 { - return plugins.NewInvalidArgError("provide non-blank secret name") + return fmt.Errorf("provide non-blank secret name") } if len(val) == 0 { - return plugins.NewInvalidArgError("provide non-blank secret value") + return fmt.Errorf("provide non-blank secret value") } return nil } + func validateSecretVersion(sec *secret.SecretVersion) error { if sec == nil { - return plugins.NewInvalidArgError("provide non-nil versioned secret") + return fmt.Errorf("provide non-nil versioned secret") } if sec.Secret == nil { - return plugins.NewInvalidArgError("provide non-nil secret") + return fmt.Errorf("provide non-nil secret") } if len(sec.Secret.Name) == 0 { - return plugins.NewInvalidArgError("provide non-blank secret name") + return fmt.Errorf("provide non-blank secret name") } if len(sec.Version) == 0 { - return plugins.NewInvalidArgError("provide non-blank secret version") + return fmt.Errorf("provide non-blank secret version") } return nil } func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { + newErr := errors.ErrorsWithScope("KeyVaultSecretService.Put") + if err := validateNewSecret(sec, val); err != nil { - return nil, err + return nil, newErr( + codes.InvalidArgument, + "invalid secret", + err, + ) } ctx := context.Background() stringVal := string(val[:]) @@ -91,7 +99,11 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec nil, ) if err != nil { - return nil, fmt.Errorf("error putting secret %v", err) + return nil, newErr( + codes.Internal, + "error putting secret", + err, + ) } return &secret.SecretPutResponse{ SecretVersion: &secret.SecretVersion{ @@ -105,8 +117,14 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec //GET https://{vaultBaseUrl}/secrets/{secret-name}/{secret-version}?api-version={api-version} func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { + newErr := errors.ErrorsWithScope("KeyVaultSecretService.Access") + if err := validateSecretVersion(sv); err != nil { - return nil, err + return nil, newErr( + codes.Internal, + "invalid secret version", + err, + ) } ctx := context.Background() result, err := s.client.Get( @@ -117,7 +135,11 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret &armkeyvault.SecretsGetOptions{}, ) if err != nil { - return nil, fmt.Errorf("error accessing secret %v", err) + return nil, newErr( + codes.Internal, + "failed to access secret", + err, + ) } return &secret.SecretAccessResponse{ // Return the original secret version payload @@ -133,13 +155,19 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret // New - Creates a new Nitric secret service with Azure Key Vault Provider func New() (secret.SecretService, error) { + newErr := errors.ErrorsWithScope("KeyVaultSecretService.New") + subscriptionId := utils.GetEnv("AZURE_SUBSCRIPTION_ID", DEFAULT_SUBSCRIPTION_ID) resouceGroup := utils.GetEnv("AZURE_RESOURCE_GROUP", DEFAULT_RESOURCE_GROUP) vaultName := utils.GetEnv("AZURE_VAULT_NAME", DEFAULT_VAULT_NAME) credentials, credentialsError := azidentity.NewDefaultAzureCredential(nil) if credentialsError != nil { - return nil, fmt.Errorf("azure credentials error: %v", credentialsError) + return nil, newErr( + codes.Internal, + "azure credentials error", + credentialsError, + ) } conn := armcore.NewDefaultConnection(credentials, nil) client := armkeyvault.NewSecretsClient(conn, subscriptionId) From b6a41b8898887b00f6846374e1b84148cb8f6648 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 19 Aug 2021 11:59:47 +1000 Subject: [PATCH 21/83] test: Fix issue with test caused by upgraded azure sdk --- go.sum | 46 +++++++++++++++++++++ pkg/plugins/gateway/appservice/http_test.go | 16 +++---- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/go.sum b/go.sum index 5523aa629..f79227921 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,10 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.5.0 h1:4qNItsmc4GP6UOZPGemmHY4ZfPofVhcaKXsYw9wm9oA= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= @@ -38,6 +40,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v55.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.2.0+incompatible h1:2GrG1JkTSMqLquy1pqVsjeRJhNtZLjss2+rx8ogZXx4= @@ -73,7 +76,9 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -91,13 +96,19 @@ github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -109,6 +120,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= @@ -118,8 +130,10 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -149,6 +163,8 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -187,6 +203,7 @@ github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab/go.mod h1:Sm/DHu github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= github.com/google/addlicense v1.0.0/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -215,7 +232,9 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2 h1:LR89qFljJ48s990kEKGsk213yIJDPI4205OKOzbURK8= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -227,11 +246,14 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7 h1:ux/56T2xqZO/3cP1I2F86qpeoYPCOzk+KF/UH/Ar+lk= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -244,15 +266,20 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -288,13 +315,17 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -310,6 +341,8 @@ github.com/spf13/afero v1.3.4 h1:8q6vk3hthlpb2SouZcnBVKboxWQWMDNF38bwholZrJc= github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -331,6 +364,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -346,6 +380,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -380,8 +415,10 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -397,6 +434,7 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -519,6 +557,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -532,6 +571,7 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -693,7 +733,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -713,7 +755,11 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/plugins/gateway/appservice/http_test.go b/pkg/plugins/gateway/appservice/http_test.go index 9a45c93d1..3d8c4964e 100644 --- a/pkg/plugins/gateway/appservice/http_test.go +++ b/pkg/plugins/gateway/appservice/http_test.go @@ -18,15 +18,16 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" - "github.com/nitric-dev/membrane/pkg/triggers" - "github.com/nitric-dev/membrane/pkg/worker" - mock_worker "github.com/nitric-dev/membrane/tests/mocks/worker" "io/ioutil" "net/http" "os" "time" + http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" + "github.com/nitric-dev/membrane/pkg/triggers" + "github.com/nitric-dev/membrane/pkg/worker" + mock_worker "github.com/nitric-dev/membrane/tests/mocks/worker" + "github.com/Azure/azure-sdk-for-go/profiles/latest/eventgrid/eventgrid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -88,11 +89,12 @@ var _ = Describe("Http", func() { When("With a SubscriptionValidation event", func() { It("Should return the provided validation code", func() { validationCode := "test" + payload := map[string]string{ + "ValidationCode": validationCode, + } evt := []eventgrid.Event{ { - Data: eventgrid.SubscriptionValidationEventData{ - ValidationCode: &validationCode, - }, + Data: payload, }, } From 7b8cf17d7519740a5d5bf8f4c103296dbbfd52e7 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 19 Aug 2021 12:04:54 +1000 Subject: [PATCH 22/83] test: Update test to reflect new error codes --- pkg/plugins/secret/key_vault/key_vault_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugins/secret/key_vault/key_vault_test.go b/pkg/plugins/secret/key_vault/key_vault_test.go index f1b1774c5..34d0fa6cb 100644 --- a/pkg/plugins/secret/key_vault/key_vault_test.go +++ b/pkg/plugins/secret/key_vault/key_vault_test.go @@ -218,7 +218,7 @@ var _ = Describe("Key Vault", func() { response, err := secretPlugin.Access(testSecretVersion) By("returning an error") Expect(err).Should(HaveOccurred()) - Expect(err.Error()).To(Equal("error accessing secret secret does not exist")) + Expect(err.Error()).To(ContainSubstring("secret does not exist")) By("returning a nil response") Expect(response).Should(BeNil()) }) From 0fd6b6d7c4888eff9c7a1d5bed4b347689f072f1 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Mon, 23 Aug 2021 19:15:28 +1000 Subject: [PATCH 23/83] feat: Update key vault to use client library instead of admin library --- go.mod | 3 +- go.sum | 4 + pkg/plugins/secret/key_vault/key_vault.go | 93 ++++++---- .../secret/key_vault/key_vault_test.go | 175 +++++++----------- 4 files changed, 123 insertions(+), 152 deletions(-) diff --git a/go.mod b/go.mod index 12db76b0d..83d5680b9 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,12 @@ require ( cloud.google.com/go/firestore v1.5.0 cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.10.0 - github.com/Azure/azure-sdk-for-go v56.2.0+incompatible + github.com/Azure/azure-sdk-for-go v56.3.0+incompatible github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2 github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1 github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/DataDog/zstd v1.4.8 // indirect github.com/Knetic/govaluate v3.0.0+incompatible diff --git a/go.sum b/go.sum index f79227921..56ea6f0ff 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/Azure/azure-sdk-for-go v55.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.2.0+incompatible h1:2GrG1JkTSMqLquy1pqVsjeRJhNtZLjss2+rx8ogZXx4= github.com/Azure/azure-sdk-for-go v56.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= +github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0 h1:HQQoaSGOh32mpoRkpLDjkngMwYJqkxu93FRx0epdLHE= github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0/go.mod h1:BSKvHb/5cy8j4hahIInXH92X/2zGJ3TxKF6b9pw1Btg= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.14.0/go.mod h1:pElNP+u99BvCZD+0jOlhI9OC/NB2IDTOTGZOZH0Qhq8= @@ -70,6 +72,8 @@ github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8K github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go index 80aaf6636..a07efe2c6 100644 --- a/pkg/plugins/secret/key_vault/key_vault.go +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -17,10 +17,10 @@ package key_vault_secret_service import ( "context" "fmt" + "strings" - "github.com/Azure/azure-sdk-for-go/sdk/armcore" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault" + "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" + "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault/keyvaultapi" "github.com/nitric-dev/membrane/pkg/plugins/errors" "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" "github.com/nitric-dev/membrane/pkg/plugins/secret" @@ -33,15 +33,11 @@ const DEFAULT_VAULT_NAME = "vault-name" // KeyVaultClient - iface that exposes utilized subset of generated KeyVaultSecretClient // Used with gomock to assert create client -> service interaction in unit tests -type KeyVaultClient interface { - Get(ctx context.Context, resourceGroupName string, vaultName string, secretName string, options *armkeyvault.SecretsGetOptions) (armkeyvault.SecretResponse, error) - CreateOrUpdate(ctx context.Context, resourceGroupName string, vaultName string, secretName string, parameters armkeyvault.SecretCreateOrUpdateParameters, options *armkeyvault.SecretsCreateOrUpdateOptions) (armkeyvault.SecretResponse, error) -} + type keyVaultSecretService struct { secret.UnimplementedSecretPlugin - client KeyVaultClient - resourceGroupName string - vaultName string + client keyvaultapi.BaseClientAPI + vaultName string } func validateNewSecret(sec *secret.Secret, val []byte) error { @@ -86,18 +82,16 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec } ctx := context.Background() stringVal := string(val[:]) - result, err := s.client.CreateOrUpdate( + + result, err := s.client.SetSecret( ctx, - s.resourceGroupName, - s.vaultName, + fmt.Sprintf("https://%s.vault.azure.net", s.vaultName), //https://myvault.vault.azure.net. sec.Name, - armkeyvault.SecretCreateOrUpdateParameters{ - Properties: &armkeyvault.SecretProperties{ - Value: &stringVal, - }, + keyvault.SecretSetParameters{ + Value: &stringVal, }, - nil, ) + if err != nil { return nil, newErr( codes.Internal, @@ -105,17 +99,21 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec err, ) } + //Returned Secret ID: https://myvault.vault.azure.net/secrets/mysecret/11a536561da34d6b8b452d880df58f3a + //Split to get the version + versionID := strings.Split(*result.ID, "/") + return &secret.SecretPutResponse{ SecretVersion: &secret.SecretVersion{ Secret: &secret.Secret{ Name: sec.Name, }, - Version: *result.Secret.Properties.SecretURIWithVersion, + Version: versionID[len(versionID)-1], }, }, nil } -//GET https://{vaultBaseUrl}/secrets/{secret-name}/{secret-version}?api-version={api-version} +//GET {vaultBaseUrl}/secrets/{secret-name}/{secret-version}?api-version={api-version} func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { newErr := errors.ErrorsWithScope("KeyVaultSecretService.Access") @@ -126,13 +124,18 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret err, ) } + ctx := context.Background() - result, err := s.client.Get( + //Key vault will default to latest if an empty string is provided + version := sv.Version + if version == "latest" { + version = "" + } + result, err := s.client.GetSecret( ctx, - s.resourceGroupName, - s.vaultName, - sv.Version, - &armkeyvault.SecretsGetOptions{}, + fmt.Sprintf("https://%s.vault.azure.net", s.vaultName), //https://myvault.vault.azure.net. + sv.Secret.Name, + version, ) if err != nil { return nil, newErr( @@ -141,15 +144,18 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret err, ) } + //Returned Secret ID: https://myvault.vault.azure.net/secrets/mysecret/11a536561da34d6b8b452d880df58f3a + //Split to get the version + versionID := strings.Split(*result.ID, "/") return &secret.SecretAccessResponse{ // Return the original secret version payload SecretVersion: &secret.SecretVersion{ Secret: &secret.Secret{ Name: sv.Secret.Name, }, - Version: *result.Secret.Properties.SecretURIWithVersion, + Version: versionID[len(versionID)-1], }, - Value: []byte(*result.Secret.Properties.Value), + Value: []byte(*result.Value), }, nil } @@ -158,23 +164,34 @@ func New() (secret.SecretService, error) { newErr := errors.ErrorsWithScope("KeyVaultSecretService.New") subscriptionId := utils.GetEnv("AZURE_SUBSCRIPTION_ID", DEFAULT_SUBSCRIPTION_ID) - resouceGroup := utils.GetEnv("AZURE_RESOURCE_GROUP", DEFAULT_RESOURCE_GROUP) vaultName := utils.GetEnv("AZURE_VAULT_NAME", DEFAULT_VAULT_NAME) - credentials, credentialsError := azidentity.NewDefaultAzureCredential(nil) - if credentialsError != nil { + if len(subscriptionId) == 0 { return nil, newErr( - codes.Internal, - "azure credentials error", - credentialsError, + codes.InvalidArgument, + "AZURE_SUBSCRIPTION_ID not configured", + fmt.Errorf(""), ) } - conn := armcore.NewDefaultConnection(credentials, nil) - client := armkeyvault.NewSecretsClient(conn, subscriptionId) + if len(vaultName) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_VAULT_NAME not configured", + fmt.Errorf(""), + ) + } + + client := keyvault.New() return &keyVaultSecretService{ - client: client, - resourceGroupName: resouceGroup, - vaultName: vaultName, + client: client, + vaultName: vaultName, }, nil } + +func NewWithClient(client keyvaultapi.BaseClientAPI) secret.SecretService { + return &keyVaultSecretService{ + client: client, + vaultName: "localvault", + } +} diff --git a/pkg/plugins/secret/key_vault/key_vault_test.go b/pkg/plugins/secret/key_vault/key_vault_test.go index 34d0fa6cb..1f1e4443e 100644 --- a/pkg/plugins/secret/key_vault/key_vault_test.go +++ b/pkg/plugins/secret/key_vault/key_vault_test.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault" + "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" "github.com/golang/mock/gomock" mocks "github.com/nitric-dev/membrane/mocks/mock_key_vault" "github.com/nitric-dev/membrane/pkg/plugins/secret" @@ -28,17 +28,13 @@ import ( var _ = Describe("Key Vault", func() { secretName := "secret-name" - secretVersion := "secret-name/version-name" + secretVersion := "secret-version" secretVal := []byte("Super Secret Message") + secretID := "https://localvault.vault.azure.net/secret/secret-name/secret-version" secretString := string(secretVal) - mockSecretResponse := armkeyvault.SecretResponse{ - Secret: &armkeyvault.Secret{ - Properties: &armkeyvault.SecretProperties{ - SecretURI: &secretName, - SecretURIWithVersion: &secretVersion, - Value: &secretString, - }, - }, + mockSecretResponse := keyvault.SecretBundle{ + ID: &secretID, + Value: &secretString, } testSecret := &secret.Secret{ Name: "secret-name", @@ -51,26 +47,19 @@ var _ = Describe("Key Vault", func() { When("Put", func() { When("Given the Key Vault backend is available", func() { When("Putting a Secret to an existing secret", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } - + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) It("Should successfully store a secret", func() { // Assert all methods are called at least their number of times - defer crtl.Finish() + defer ctrl.Finish() //Mocking expects - mockSecretClient.EXPECT().CreateOrUpdate( + mockSecretClient.EXPECT().SetSecret( context.Background(), - secretPlugin.resourceGroupName, - secretPlugin.vaultName, + "https://localvault.vault.azure.net", testSecret.Name, gomock.Any(), - gomock.Any(), ).Return(mockSecretResponse, nil).Times(1) response, err := secretPlugin.Put(testSecret, secretVal) @@ -83,24 +72,18 @@ var _ = Describe("Key Vault", func() { }) When("Putting a Secret to a non-existent secret", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) It("Should successfully store a secret", func() { - defer crtl.Finish() + defer ctrl.Finish() //Mocking expects - mockSecretClient.EXPECT().CreateOrUpdate( + mockSecretClient.EXPECT().SetSecret( context.Background(), - secretPlugin.resourceGroupName, - secretPlugin.vaultName, + "https://localvault.vault.azure.net", testSecret.Name, gomock.Any(), - gomock.Any(), ).Return(mockSecretResponse, nil).Times(1) response, err := secretPlugin.Put(testSecret, secretVal) @@ -113,13 +96,9 @@ var _ = Describe("Key Vault", func() { }) When("Putting a nil secret", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) It("Should invalidate the secret", func() { _, err := secretPlugin.Put(nil, secretVal) @@ -129,13 +108,9 @@ var _ = Describe("Key Vault", func() { }) When("Putting a secret with an empty name", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) It("Should invalidate the secret", func() { _, err := secretPlugin.Put(&secret.Secret{Name: ""}, secretVal) @@ -145,13 +120,9 @@ var _ = Describe("Key Vault", func() { }) When("Putting a secret with an empty value", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) It("Should invalidate the secret", func() { _, err := secretPlugin.Put(testSecret, nil) @@ -166,23 +137,18 @@ var _ = Describe("Key Vault", func() { When("Given the Key Vault backend is available", func() { When("The secret store exists", func() { When("The secret exists", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) It("Should successfully return a secret", func() { - defer crtl.Finish() + defer ctrl.Finish() //Mocking expects - mockSecretClient.EXPECT().Get( + mockSecretClient.EXPECT().GetSecret( context.Background(), - secretPlugin.resourceGroupName, - secretPlugin.vaultName, + "https://localvault.vault.azure.net", + secretName, secretVersion, - gomock.Any(), ).Return(mockSecretResponse, nil).Times(1) response, err := secretPlugin.Access(testSecretVersion) @@ -197,23 +163,18 @@ var _ = Describe("Key Vault", func() { }) }) When("The secret doesn't exist", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) It("Should return an error", func() { - defer crtl.Finish() + defer ctrl.Finish() - mockSecretClient.EXPECT().Get( + mockSecretClient.EXPECT().GetSecret( context.Background(), - secretPlugin.resourceGroupName, - secretPlugin.vaultName, + "https://localvault.vault.azure.net", + secretName, secretVersion, - gomock.Any(), - ).Return(armkeyvault.SecretResponse{}, fmt.Errorf("secret does not exist")).Times(1) + ).Return(keyvault.SecretBundle{}, fmt.Errorf("secret does not exist")).Times(1) response, err := secretPlugin.Access(testSecretVersion) By("returning an error") @@ -224,15 +185,12 @@ var _ = Describe("Key Vault", func() { }) }) When("An empty secret version is provided", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + It("Should return an error", func() { - defer crtl.Finish() + defer ctrl.Finish() response, err := secretPlugin.Access(nil) By("returning an error") @@ -242,15 +200,12 @@ var _ = Describe("Key Vault", func() { }) }) When("An empty secret is provided", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + It("Should return an error", func() { - defer crtl.Finish() + defer ctrl.Finish() response, err := secretPlugin.Access(&secret.SecretVersion{Secret: nil, Version: secretVersion}) By("returning an error") @@ -260,15 +215,12 @@ var _ = Describe("Key Vault", func() { }) }) When("An empty secret name is provided", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + It("Should return an error", func() { - defer crtl.Finish() + defer ctrl.Finish() response, err := secretPlugin.Access(&secret.SecretVersion{Secret: &secret.Secret{Name: ""}, Version: secretVersion}) By("returning an error") @@ -278,15 +230,12 @@ var _ = Describe("Key Vault", func() { }) }) When("An empty version is provided", func() { - crtl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockKeyVaultClient(crtl) - secretPlugin := &keyVaultSecretService{ - client: mockSecretClient, - resourceGroupName: "resource-group-name", - vaultName: "vault-name", - } + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + It("Should return an error", func() { - defer crtl.Finish() + defer ctrl.Finish() response, err := secretPlugin.Access(&secret.SecretVersion{Secret: testSecret, Version: ""}) By("returning an error") From 275f69c9a1b44cf68011b34ce625cd510539f001 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 24 Aug 2021 13:10:09 +1000 Subject: [PATCH 24/83] feat: Add access token for key vault requests --- pkg/plugins/secret/key_vault/key_vault.go | 100 ++++++++++++++++++---- 1 file changed, 84 insertions(+), 16 deletions(-) diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go index a07efe2c6..cb148c3ac 100644 --- a/pkg/plugins/secret/key_vault/key_vault.go +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -16,7 +16,10 @@ package key_vault_secret_service import ( "context" + "encoding/json" "fmt" + "net/http" + "net/url" "strings" "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" @@ -27,17 +30,41 @@ import ( "github.com/nitric-dev/membrane/pkg/utils" ) -const DEFAULT_SUBSCRIPTION_ID = "subscription-id" -const DEFAULT_RESOURCE_GROUP = "resource-group" -const DEFAULT_VAULT_NAME = "vault-name" - -// KeyVaultClient - iface that exposes utilized subset of generated KeyVaultSecretClient -// Used with gomock to assert create client -> service interaction in unit tests - type keyVaultSecretService struct { secret.UnimplementedSecretPlugin - client keyvaultapi.BaseClientAPI - vaultName string + client keyvaultapi.BaseClientAPI + accessToken AzureAccessToken + vaultName string +} + +type AzureAccessToken struct { + TokenType string `json:"token_type"` + ExpiresIn string `json:"expires_in"` + ExtExpiresIn string `json:"ext_expires_in"` + ExpiresOn string `json:"expires_on"` + NotBefore string `json:"not_before"` + Resource string `json:"resource"` + AccessToken string `json:"access_token"` +} + +func GetToken(tenantId string, clientId string, clientSecret string) (AzureAccessToken, error) { + requestAccessTokenUri := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/token", tenantId) + requestBody := url.Values{ + "grant_type": {"client_credentials"}, + "client_id": {clientId}, + "client_secret": {clientSecret}, + "resource": {"https://management.azure.com/"}, + } + resp, err := http.PostForm(requestAccessTokenUri, requestBody) + if err != nil { + return AzureAccessToken{}, err + } + + var result AzureAccessToken + + json.NewDecoder(resp.Body).Decode(&result) + + return result, nil } func validateNewSecret(sec *secret.Secret, val []byte) error { @@ -81,6 +108,9 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec ) } ctx := context.Background() + ctx.Value(map[string]string{ + "Authorization": s.accessToken.TokenType + " " + s.accessToken.AccessToken, + }) stringVal := string(val[:]) result, err := s.client.SetSecret( @@ -99,7 +129,7 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec err, ) } - //Returned Secret ID: https://myvault.vault.azure.net/secrets/mysecret/11a536561da34d6b8b452d880df58f3a + //Returned Secret ID: https://myvault.vault.azure.net/secrets/{SECRET_NAME}/{SECRET_VERSION} //Split to get the version versionID := strings.Split(*result.ID, "/") @@ -113,7 +143,6 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec }, nil } -//GET {vaultBaseUrl}/secrets/{secret-name}/{secret-version}?api-version={api-version} func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { newErr := errors.ErrorsWithScope("KeyVaultSecretService.Access") @@ -126,6 +155,10 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret } ctx := context.Background() + ctx.Value(map[string]string{ + "Authorization": fmt.Sprintf("%s %s", s.accessToken.TokenType, s.accessToken.AccessToken), + }) + //Key vault will default to latest if an empty string is provided version := sv.Version if version == "latest" { @@ -163,8 +196,35 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret func New() (secret.SecretService, error) { newErr := errors.ErrorsWithScope("KeyVaultSecretService.New") - subscriptionId := utils.GetEnv("AZURE_SUBSCRIPTION_ID", DEFAULT_SUBSCRIPTION_ID) - vaultName := utils.GetEnv("AZURE_VAULT_NAME", DEFAULT_VAULT_NAME) + subscriptionId := utils.GetEnv("AZURE_SUBSCRIPTION_ID", "") + vaultName := utils.GetEnv("AZURE_VAULT_NAME", "") + tenantId := utils.GetEnv("AZURE_TENANT_ID", "") + clientId := utils.GetEnv("AZURE_CLIENT_ID", "") + clientSecret := utils.GetEnv("AZURE_CLIENT_SECRET", "") + + if len(tenantId) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_TENANT_ID not configured", + fmt.Errorf(""), + ) + } + + if len(clientId) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_CLIENT_ID not configured", + fmt.Errorf(""), + ) + } + + if len(clientSecret) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_CLIENT_SECRET not configured", + fmt.Errorf(""), + ) + } if len(subscriptionId) == 0 { return nil, newErr( @@ -182,10 +242,18 @@ func New() (secret.SecretService, error) { } client := keyvault.New() - + token, err := GetToken(tenantId, clientId, clientSecret) + if err != nil { + return nil, newErr( + codes.Unauthenticated, + "Error authenticating key vault", + err, + ) + } return &keyVaultSecretService{ - client: client, - vaultName: vaultName, + client: client, + vaultName: vaultName, + accessToken: token, }, nil } From fb187f6b88b93a27d7c646f374f9700f7ae4ff52 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 24 Aug 2021 15:20:35 +1000 Subject: [PATCH 25/83] feat: Add client auth --- go.mod | 1 + go.sum | 16 +++ pkg/plugins/secret/key_vault/key_vault.go | 119 ++++------------------ 3 files changed, 35 insertions(+), 101 deletions(-) diff --git a/go.mod b/go.mod index 83d5680b9..0f33fa769 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2 github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1 github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/DataDog/zstd v1.4.8 // indirect diff --git a/go.sum b/go.sum index 56ea6f0ff..82dda3b84 100644 --- a/go.sum +++ b/go.sum @@ -64,10 +64,17 @@ github.com/Azure/azure-sdk-for-go/sdk/to v0.1.4 h1:3w4gk+uYOwplGhID1fDP305/8bI5A github.com/Azure/azure-sdk-for-go/sdk/to v0.1.4/go.mod h1:UL/d4lvWAzSJUuX+19uKdN0ktyjoOyQhgY+HWNgtIYI= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -76,6 +83,7 @@ github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+X github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= @@ -120,6 +128,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -297,6 +308,8 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= @@ -410,6 +423,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -485,6 +500,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go index cb148c3ac..f84791f18 100644 --- a/pkg/plugins/secret/key_vault/key_vault.go +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -16,12 +16,10 @@ package key_vault_secret_service import ( "context" - "encoding/json" "fmt" - "net/http" - "net/url" "strings" + kvauth "github.com/Azure/azure-sdk-for-go/services/keyvault/auth" "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault/keyvaultapi" "github.com/nitric-dev/membrane/pkg/plugins/errors" @@ -32,39 +30,8 @@ import ( type keyVaultSecretService struct { secret.UnimplementedSecretPlugin - client keyvaultapi.BaseClientAPI - accessToken AzureAccessToken - vaultName string -} - -type AzureAccessToken struct { - TokenType string `json:"token_type"` - ExpiresIn string `json:"expires_in"` - ExtExpiresIn string `json:"ext_expires_in"` - ExpiresOn string `json:"expires_on"` - NotBefore string `json:"not_before"` - Resource string `json:"resource"` - AccessToken string `json:"access_token"` -} - -func GetToken(tenantId string, clientId string, clientSecret string) (AzureAccessToken, error) { - requestAccessTokenUri := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/token", tenantId) - requestBody := url.Values{ - "grant_type": {"client_credentials"}, - "client_id": {clientId}, - "client_secret": {clientSecret}, - "resource": {"https://management.azure.com/"}, - } - resp, err := http.PostForm(requestAccessTokenUri, requestBody) - if err != nil { - return AzureAccessToken{}, err - } - - var result AzureAccessToken - - json.NewDecoder(resp.Body).Decode(&result) - - return result, nil + client keyvaultapi.BaseClientAPI + vaultName string } func validateNewSecret(sec *secret.Secret, val []byte) error { @@ -107,14 +74,10 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec err, ) } - ctx := context.Background() - ctx.Value(map[string]string{ - "Authorization": s.accessToken.TokenType + " " + s.accessToken.AccessToken, - }) stringVal := string(val[:]) result, err := s.client.SetSecret( - ctx, + context.Background(), fmt.Sprintf("https://%s.vault.azure.net", s.vaultName), //https://myvault.vault.azure.net. sec.Name, keyvault.SecretSetParameters{ @@ -154,18 +117,13 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret ) } - ctx := context.Background() - ctx.Value(map[string]string{ - "Authorization": fmt.Sprintf("%s %s", s.accessToken.TokenType, s.accessToken.AccessToken), - }) - //Key vault will default to latest if an empty string is provided version := sv.Version if version == "latest" { version = "" } result, err := s.client.GetSecret( - ctx, + context.Background(), fmt.Sprintf("https://%s.vault.azure.net", s.vaultName), //https://myvault.vault.azure.net. sv.Secret.Name, version, @@ -194,66 +152,25 @@ func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret // New - Creates a new Nitric secret service with Azure Key Vault Provider func New() (secret.SecretService, error) { - newErr := errors.ErrorsWithScope("KeyVaultSecretService.New") - - subscriptionId := utils.GetEnv("AZURE_SUBSCRIPTION_ID", "") - vaultName := utils.GetEnv("AZURE_VAULT_NAME", "") - tenantId := utils.GetEnv("AZURE_TENANT_ID", "") - clientId := utils.GetEnv("AZURE_CLIENT_ID", "") - clientSecret := utils.GetEnv("AZURE_CLIENT_SECRET", "") - - if len(tenantId) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_TENANT_ID not configured", - fmt.Errorf(""), - ) - } - - if len(clientId) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_CLIENT_ID not configured", - fmt.Errorf(""), - ) - } - - if len(clientSecret) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_CLIENT_SECRET not configured", - fmt.Errorf(""), - ) - } - - if len(subscriptionId) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_SUBSCRIPTION_ID not configured", - fmt.Errorf(""), - ) - } + vaultName := utils.GetEnv("KVAULT_NAME", "") if len(vaultName) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_VAULT_NAME not configured", - fmt.Errorf(""), - ) + return nil, fmt.Errorf("KVAULT_NAME not configured") } - client := keyvault.New() - token, err := GetToken(tenantId, clientId, clientSecret) + //Auth requires: + //AZURE_TENANT_ID: Your Azure tenant ID + //AZURE_CLIENT_ID: Your Azure client ID. This will be an app ID from your AAD. + authorizer, err := kvauth.NewAuthorizerFromEnvironment() if err != nil { - return nil, newErr( - codes.Unauthenticated, - "Error authenticating key vault", - err, - ) + return nil, err } + + client := keyvault.New() + client.Authorizer = authorizer + return &keyVaultSecretService{ - client: client, - vaultName: vaultName, - accessToken: token, + client: client, + vaultName: vaultName, }, nil } From 2ff4a59a7807f15f28600c2677f99b394bd0ee32 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Mon, 30 Aug 2021 11:17:47 +1000 Subject: [PATCH 26/83] chore: Update mocking interface --- go.mod | 5 +- go.sum | 59 ------------------- pkg/plugins/secret/key_vault/key_vault.go | 20 ++++--- .../secret/key_vault/key_vault_test.go | 22 +++---- 4 files changed, 24 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index 0f33fa769..a30b3e9c1 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,7 @@ require ( cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.10.0 github.com/Azure/azure-sdk-for-go v56.3.0+incompatible - github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2 - github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1 - github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest v0.11.18 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect diff --git a/go.sum b/go.sum index 82dda3b84..7e949517a 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,8 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.5.0 h1:4qNItsmc4GP6UOZPGemmHY4ZfPofVhcaKXsYw9wm9oA= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= @@ -40,28 +38,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v55.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v56.2.0+incompatible h1:2GrG1JkTSMqLquy1pqVsjeRJhNtZLjss2+rx8ogZXx4= -github.com/Azure/azure-sdk-for-go v56.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0 h1:HQQoaSGOh32mpoRkpLDjkngMwYJqkxu93FRx0epdLHE= -github.com/Azure/azure-sdk-for-go/sdk/armcore v0.8.0/go.mod h1:BSKvHb/5cy8j4hahIInXH92X/2zGJ3TxKF6b9pw1Btg= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.14.0/go.mod h1:pElNP+u99BvCZD+0jOlhI9OC/NB2IDTOTGZOZH0Qhq8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.16.2 h1:UC4vfOhW2l0f2QOCQpOxJS4/K6oKFy2tQZE+uWU1MEo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.16.2/go.mod h1:MVdrcUC4Hup35qHym3VdzoW+NBgBxrta9Vei97jRtM8= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.1/go.mod h1:acANgl9stsT5xflESXKjZx4rhZJSr0TGgTDYY0xJPIE= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2 h1:3W8umQHRg0DXV5KvmbqU43uFi8MKvqLHQCE8L8v6Xds= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.9.2/go.mod h1:acANgl9stsT5xflESXKjZx4rhZJSr0TGgTDYY0xJPIE= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.5.0/go.mod h1:k4KbFSunV/+0hOHL1vyFaPsiYQ1Vmvy1TBpmtvCDLZM= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.5.1 h1:vx8McI56N5oLSQu8xa+xdiE0fjQq8W8Zt49vHP8Rygw= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.5.1/go.mod h1:k4KbFSunV/+0hOHL1vyFaPsiYQ1Vmvy1TBpmtvCDLZM= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1 h1:GZP4qFuNKtt7Mq8PY2zCsgJOO0mqa6cyXZb0u1q+EH8= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/armkeyvault v0.1.1/go.mod h1:2TVZ9s3y3yhuyfHnId/SjaZ2Wly3EPcD7SvzYubFdrM= -github.com/Azure/azure-sdk-for-go/sdk/to v0.1.4 h1:3w4gk+uYOwplGhID1fDP305/8bI5Aug3URoC1V493KU= -github.com/Azure/azure-sdk-for-go/sdk/to v0.1.4/go.mod h1:UL/d4lvWAzSJUuX+19uKdN0ktyjoOyQhgY+HWNgtIYI= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= @@ -88,9 +67,7 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -108,19 +85,13 @@ github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -135,7 +106,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= @@ -145,10 +115,8 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -213,12 +181,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab h1:+qfOxKbnAqDNCoFUNHxudKs8Z14T5EBYntAeWIeI1eA= -github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= github.com/google/addlicense v1.0.0/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -247,9 +212,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2 h1:LR89qFljJ48s990kEKGsk213yIJDPI4205OKOzbURK8= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -261,14 +224,11 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7 h1:ux/56T2xqZO/3cP1I2F86qpeoYPCOzk+KF/UH/Ar+lk= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -294,7 +254,6 @@ github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -332,12 +291,10 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -381,7 +338,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -397,7 +353,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -420,8 +375,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= @@ -434,10 +387,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -453,7 +404,6 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -493,7 +443,6 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -577,7 +526,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -591,7 +539,6 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -753,9 +700,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -775,11 +720,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go index f84791f18..090b463b7 100644 --- a/pkg/plugins/secret/key_vault/key_vault.go +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -21,16 +21,20 @@ import ( kvauth "github.com/Azure/azure-sdk-for-go/services/keyvault/auth" "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" - "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault/keyvaultapi" "github.com/nitric-dev/membrane/pkg/plugins/errors" "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" "github.com/nitric-dev/membrane/pkg/plugins/secret" "github.com/nitric-dev/membrane/pkg/utils" ) -type keyVaultSecretService struct { +type KeyVaultClient interface { + SetSecret(ctx context.Context, vaultBaseURL string, secretName string, parameters keyvault.SecretSetParameters) (result keyvault.SecretBundle, err error) + GetSecret(ctx context.Context, vaultBaseURL string, secretName string, secretVersion string) (result keyvault.SecretBundle, err error) +} + +type KeyVaultSecretService struct { secret.UnimplementedSecretPlugin - client keyvaultapi.BaseClientAPI + client KeyVaultClient vaultName string } @@ -64,7 +68,7 @@ func validateSecretVersion(sec *secret.SecretVersion) error { return nil } -func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { +func (s *KeyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { newErr := errors.ErrorsWithScope("KeyVaultSecretService.Put") if err := validateNewSecret(sec, val); err != nil { @@ -106,7 +110,7 @@ func (s *keyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec }, nil } -func (s *keyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { +func (s *KeyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { newErr := errors.ErrorsWithScope("KeyVaultSecretService.Access") if err := validateSecretVersion(sv); err != nil { @@ -168,14 +172,14 @@ func New() (secret.SecretService, error) { client := keyvault.New() client.Authorizer = authorizer - return &keyVaultSecretService{ + return &KeyVaultSecretService{ client: client, vaultName: vaultName, }, nil } -func NewWithClient(client keyvaultapi.BaseClientAPI) secret.SecretService { - return &keyVaultSecretService{ +func NewWithClient(client KeyVaultClient) secret.SecretService { + return &KeyVaultSecretService{ client: client, vaultName: "localvault", } diff --git a/pkg/plugins/secret/key_vault/key_vault_test.go b/pkg/plugins/secret/key_vault/key_vault_test.go index 1f1e4443e..ee068779c 100644 --- a/pkg/plugins/secret/key_vault/key_vault_test.go +++ b/pkg/plugins/secret/key_vault/key_vault_test.go @@ -48,7 +48,7 @@ var _ = Describe("Key Vault", func() { When("Given the Key Vault backend is available", func() { When("Putting a Secret to an existing secret", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should successfully store a secret", func() { // Assert all methods are called at least their number of times @@ -73,7 +73,7 @@ var _ = Describe("Key Vault", func() { When("Putting a Secret to a non-existent secret", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should successfully store a secret", func() { defer ctrl.Finish() @@ -97,7 +97,7 @@ var _ = Describe("Key Vault", func() { When("Putting a nil secret", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should invalidate the secret", func() { @@ -109,7 +109,7 @@ var _ = Describe("Key Vault", func() { When("Putting a secret with an empty name", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should invalidate the secret", func() { @@ -121,7 +121,7 @@ var _ = Describe("Key Vault", func() { When("Putting a secret with an empty value", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should invalidate the secret", func() { @@ -138,7 +138,7 @@ var _ = Describe("Key Vault", func() { When("The secret store exists", func() { When("The secret exists", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should successfully return a secret", func() { @@ -164,7 +164,7 @@ var _ = Describe("Key Vault", func() { }) When("The secret doesn't exist", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should return an error", func() { defer ctrl.Finish() @@ -186,7 +186,7 @@ var _ = Describe("Key Vault", func() { }) When("An empty secret version is provided", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should return an error", func() { @@ -201,7 +201,7 @@ var _ = Describe("Key Vault", func() { }) When("An empty secret is provided", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should return an error", func() { @@ -216,7 +216,7 @@ var _ = Describe("Key Vault", func() { }) When("An empty secret name is provided", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should return an error", func() { @@ -231,7 +231,7 @@ var _ = Describe("Key Vault", func() { }) When("An empty version is provided", func() { ctrl := gomock.NewController(GinkgoT()) - mockSecretClient := mocks.NewMockBaseClientAPI(ctrl) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) secretPlugin := NewWithClient(mockSecretClient) It("Should return an error", func() { From 4711fcd30ae609744314edfc60e62844f3a5de7a Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 14 Sep 2021 15:55:54 +1000 Subject: [PATCH 27/83] feat: Add key vault to azure membrane --- go.sum | 63 ++++++++++++++++++++++++++++++++- pkg/membrane/membrane.go | 8 ++--- pkg/providers/azure/membrane.go | 6 ++++ pkg/providers/azure/plugin.go | 6 ++++ 4 files changed, 78 insertions(+), 5 deletions(-) diff --git a/go.sum b/go.sum index 7e949517a..da7ab82df 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,10 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.5.0 h1:4qNItsmc4GP6UOZPGemmHY4ZfPofVhcaKXsYw9wm9oA= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= @@ -38,6 +40,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -67,7 +70,9 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -85,13 +90,19 @@ github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -106,6 +117,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= @@ -115,36 +127,50 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3 h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -184,6 +210,7 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= github.com/google/addlicense v1.0.0/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -212,7 +239,9 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2 h1:LR89qFljJ48s990kEKGsk213yIJDPI4205OKOzbURK8= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -224,22 +253,28 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7 h1:ux/56T2xqZO/3cP1I2F86qpeoYPCOzk+KF/UH/Ar+lk= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= @@ -248,12 +283,14 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -261,7 +298,9 @@ github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tW github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-star v0.5.1 h1:sImehRT+p7lW9n6R7MQc5hVgzWGEkDVZU4AsBQ4Isu8= github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -271,6 +310,7 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/muesli/termenv v0.7.4 h1:/pBqvU5CpkY53tU0vVn+xgs2ZTX63aH5nY+SSps5Xa8= github.com/muesli/termenv v0.7.4/go.mod h1:pZ7qY9l3F7e5xsAOS0zCew2tME+p7bWeBkotCEcIIcc= @@ -286,15 +326,19 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -309,15 +353,19 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.3.4 h1:8q6vk3hthlpb2SouZcnBVKboxWQWMDNF38bwholZrJc= github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -338,6 +386,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -353,6 +402,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -387,8 +437,10 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -404,6 +456,7 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -526,6 +579,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -539,6 +593,7 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -700,7 +755,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -720,7 +777,11 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/membrane/membrane.go b/pkg/membrane/membrane.go index 5f758a00a..15d03fd3f 100644 --- a/pkg/membrane/membrane.go +++ b/pkg/membrane/membrane.go @@ -109,7 +109,7 @@ func (s *Membrane) log(log string) { } } -func (s *Membrane) CreateSecretServer() v1.SecretServiceServer { +func (s *Membrane) createSecretServer() v1.SecretServiceServer { return grpc2.NewSecretServer(s.secretPlugin) } @@ -162,9 +162,6 @@ func (s *Membrane) Start() error { var opts []grpc.ServerOption s.grpcServer = grpc.NewServer(opts...) - secretServer := s.CreateSecretServer() - v1.RegisterSecretServiceServer(s.grpcServer, secretServer) - // Load & Register the GRPC service plugins documentServer := s.createDocumentServer() v1.RegisterDocumentServiceServer(s.grpcServer, documentServer) @@ -181,6 +178,9 @@ func (s *Membrane) Start() error { queueServer := s.createQueueServer() v1.RegisterQueueServiceServer(s.grpcServer, queueServer) + secretServer := s.createSecretServer() + v1.RegisterSecretServiceServer(s.grpcServer, secretServer) + // FaaS server MUST start before the child process if s.mode == Mode_Faas { faasServer := grpc2.NewFaasServer(s.pool) diff --git a/pkg/providers/azure/membrane.go b/pkg/providers/azure/membrane.go index e539806bd..1503ef036 100644 --- a/pkg/providers/azure/membrane.go +++ b/pkg/providers/azure/membrane.go @@ -26,6 +26,7 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/events" http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" "github.com/nitric-dev/membrane/pkg/plugins/queue" + key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" "github.com/nitric-dev/membrane/pkg/plugins/storage" ) @@ -43,6 +44,10 @@ func main() { gatewayPlugin, _ := http_service.New() queuePlugin := &queue.UnimplementedQueuePlugin{} storagePlugin := &storage.UnimplementedStoragePlugin{} + secretPlugin, err := key_vault.New() + if err != nil { + fmt.Println("Failed to load secret plugin:", err.Error()) + } m, err := membrane.New(&membrane.MembraneOptions{ DocumentPlugin: documentPlugin, @@ -50,6 +55,7 @@ func main() { GatewayPlugin: gatewayPlugin, QueuePlugin: queuePlugin, StoragePlugin: storagePlugin, + SecretPlugin: secretPlugin, }) if err != nil { diff --git a/pkg/providers/azure/plugin.go b/pkg/providers/azure/plugin.go index 56d8b016d..85b452cfa 100644 --- a/pkg/providers/azure/plugin.go +++ b/pkg/providers/azure/plugin.go @@ -21,6 +21,8 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/gateway" http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" "github.com/nitric-dev/membrane/pkg/plugins/queue" + "github.com/nitric-dev/membrane/pkg/plugins/secret" + key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" "github.com/nitric-dev/membrane/pkg/plugins/storage" "github.com/nitric-dev/membrane/pkg/providers" ) @@ -32,6 +34,10 @@ func New() providers.ServiceFactory { return &AzureServiceFactory{} } +func (p *AzureServiceFactory) NewSecretService() (secret.SecretService, error) { + return key_vault.New() +} + // NewDocumentService - Returns a MongoDB based document service func (p *AzureServiceFactory) NewDocumentService() (document.DocumentService, error) { return mongodb_service.New() From 61f457b7033208c1222ccd1e60d6b346523557db Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 16 Sep 2021 11:13:48 +1000 Subject: [PATCH 28/83] chore: Fix merge conflicts, test cases and update auth syntax --- go.mod | 1 + pkg/plugins/secret/key_vault/key_vault.go | 40 ++++++++++++++---- pkg/providers/azure/utils/auth.go | 50 +++++++++++++++++++++++ 3 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 pkg/providers/azure/utils/auth.go diff --git a/go.mod b/go.mod index a30b3e9c1..7f96612cb 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( cloud.google.com/go/storage v1.10.0 github.com/Azure/azure-sdk-for-go v56.3.0+incompatible github.com/Azure/go-autorest/autorest v0.11.18 + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go index 090b463b7..5f5bfd6fe 100644 --- a/pkg/plugins/secret/key_vault/key_vault.go +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -19,11 +19,13 @@ import ( "fmt" "strings" - kvauth "github.com/Azure/azure-sdk-for-go/services/keyvault/auth" "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" "github.com/nitric-dev/membrane/pkg/plugins/errors" "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" "github.com/nitric-dev/membrane/pkg/plugins/secret" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" "github.com/nitric-dev/membrane/pkg/utils" ) @@ -69,15 +71,25 @@ func validateSecretVersion(sec *secret.SecretVersion) error { } func (s *KeyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { - newErr := errors.ErrorsWithScope("KeyVaultSecretService.Put") - + validationErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Put", + map[string]interface{}{ + "secret": "nil", + }, + ) if err := validateNewSecret(sec, val); err != nil { - return nil, newErr( + return nil, validationErr( codes.InvalidArgument, "invalid secret", err, ) } + newErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Put", + map[string]interface{}{ + "secret": sec.Name, + }, + ) stringVal := string(val[:]) result, err := s.client.SetSecret( @@ -111,15 +123,25 @@ func (s *KeyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec } func (s *KeyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { - newErr := errors.ErrorsWithScope("KeyVaultSecretService.Access") - + validationErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Access", + map[string]interface{}{ + "secret-version": "nil", + }, + ) if err := validateSecretVersion(sv); err != nil { - return nil, newErr( + return nil, validationErr( codes.Internal, "invalid secret version", err, ) } + newErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Access", + map[string]interface{}{ + "secret-version": sv.Secret.Name, + }, + ) //Key vault will default to latest if an empty string is provided version := sv.Version @@ -164,13 +186,13 @@ func New() (secret.SecretService, error) { //Auth requires: //AZURE_TENANT_ID: Your Azure tenant ID //AZURE_CLIENT_ID: Your Azure client ID. This will be an app ID from your AAD. - authorizer, err := kvauth.NewAuthorizerFromEnvironment() + spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.KeyVault) if err != nil { return nil, err } client := keyvault.New() - client.Authorizer = authorizer + client.Authorizer = autorest.NewBearerAuthorizer(spt) return &KeyVaultSecretService{ client: client, diff --git a/pkg/providers/azure/utils/auth.go b/pkg/providers/azure/utils/auth.go new file mode 100644 index 000000000..6b5c94325 --- /dev/null +++ b/pkg/providers/azure/utils/auth.go @@ -0,0 +1,50 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure/auth" +) + +// GetServicePrincipalToken - Retrieves the service principal token from env +func GetServicePrincipalToken(resource string) (*adal.ServicePrincipalToken, error) { + config, err := auth.GetSettingsFromEnvironment() + + if err != nil { + return nil, fmt.Errorf("failed to retrieve azure auth settings: %v", err) + } + + if fileCred, fileErr := auth.GetSettingsFromFile(); fileErr == nil { + return fileCred.ServicePrincipalTokenFromClientCredentialsWithResource(resource) + } else if clientCred, clientErr := config.GetClientCredentials(); clientErr == nil { + clientCred.Resource = resource + return clientCred.ServicePrincipalToken() + } else if clientCert, certErr := config.GetClientCertificate(); certErr == nil { + clientCert.Resource = resource + return clientCert.ServicePrincipalToken() + } else if userPass, userErr := config.GetUsernamePassword(); userErr == nil { + userPass.Resource = resource + return userPass.ServicePrincipalToken() + } else { + fmt.Printf("error retrieving credentials:\n -> %v\n -> %v\n -> %v\n -> %v\n", fileErr, clientErr, certErr, userErr) + } + + msiConf := config.GetMSI() + msiConf.Resource = resource + return msiConf.ServicePrincipalToken() +} From ea5af90e279e837adbe3dd45b7500e198fe062dc Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 16 Sep 2021 23:23:16 +1000 Subject: [PATCH 29/83] chore: split out version id extraction into separate function --- pkg/plugins/secret/key_vault/key_vault.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go index 5f5bfd6fe..7fc275e2f 100644 --- a/pkg/plugins/secret/key_vault/key_vault.go +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -40,6 +40,13 @@ type KeyVaultSecretService struct { vaultName string } +// versionIdFromUrl - Extracts a secret version ID from a full secret version URL +// the expected versionUrl format is https://{VAULT_NAME}.vault.azure.net/secrets/{SECRET_NAME}/{SECRET_VERSION} +func versionIdFromUrl(versionUrl string) string { + urlParts := strings.Split(versionUrl, "/") + return urlParts[len(urlParts)-1] +} + func validateNewSecret(sec *secret.Secret, val []byte) error { if sec == nil { return fmt.Errorf("provide non-nil secret") @@ -108,16 +115,13 @@ func (s *KeyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.Sec err, ) } - //Returned Secret ID: https://myvault.vault.azure.net/secrets/{SECRET_NAME}/{SECRET_VERSION} - //Split to get the version - versionID := strings.Split(*result.ID, "/") return &secret.SecretPutResponse{ SecretVersion: &secret.SecretVersion{ Secret: &secret.Secret{ Name: sec.Name, }, - Version: versionID[len(versionID)-1], + Version: versionIdFromUrl(*result.ID), }, }, nil } @@ -163,14 +167,13 @@ func (s *KeyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.Secret } //Returned Secret ID: https://myvault.vault.azure.net/secrets/mysecret/11a536561da34d6b8b452d880df58f3a //Split to get the version - versionID := strings.Split(*result.ID, "/") return &secret.SecretAccessResponse{ // Return the original secret version payload SecretVersion: &secret.SecretVersion{ Secret: &secret.Secret{ Name: sv.Secret.Name, }, - Version: versionID[len(versionID)-1], + Version: versionIdFromUrl(*result.ID), }, Value: []byte(*result.Value), }, nil From 76c515248a520fb31bf6150729d66de9e086950b Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Fri, 17 Sep 2021 11:23:51 +1000 Subject: [PATCH 30/83] chore: Fix merge conflicts --- makefile | 12 ++++++------ pkg/plugins/secret/key_vault/key_vault_test.go | 2 +- .../secret/secret_manager/secret_manager_test.go | 2 +- .../secret/secrets_manager/secrets_manager_test.go | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/makefile b/makefile index c5ddaec59..d0c2732cf 100644 --- a/makefile +++ b/makefile @@ -192,11 +192,11 @@ build-all-binaries: clean generate-proto # generate mock implementations generate-mocks: @echo Generating Mock Clients - @mkdir -p mocks/mock_secret_manager - @mkdir -p mocks/mock_secrets_manager - @mkdir -p mocks/mock_key_vault + @mkdir -p mocks/secret_manager + @mkdir -p mocks/secrets_manager + @mkdir -p mocks/key_vault @mkdir -p mocks/s3 - @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/mock_secret_manager/mock.go - @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/mock_secrets_manager/mock.go - @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/mock_key_vault/mock.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/secret_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/key_vault/mock.go @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go \ No newline at end of file diff --git a/pkg/plugins/secret/key_vault/key_vault_test.go b/pkg/plugins/secret/key_vault/key_vault_test.go index ee068779c..3a69dca62 100644 --- a/pkg/plugins/secret/key_vault/key_vault_test.go +++ b/pkg/plugins/secret/key_vault/key_vault_test.go @@ -20,7 +20,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" "github.com/golang/mock/gomock" - mocks "github.com/nitric-dev/membrane/mocks/mock_key_vault" + mocks "github.com/nitric-dev/membrane/mocks/key_vault" "github.com/nitric-dev/membrane/pkg/plugins/secret" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/plugins/secret/secret_manager/secret_manager_test.go b/pkg/plugins/secret/secret_manager/secret_manager_test.go index f2dd00896..5b9cce9f7 100644 --- a/pkg/plugins/secret/secret_manager/secret_manager_test.go +++ b/pkg/plugins/secret/secret_manager/secret_manager_test.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/golang/mock/gomock" - mocks "github.com/nitric-dev/membrane/mocks/mock_secret_manager" + mocks "github.com/nitric-dev/membrane/mocks/secret_manager" "github.com/nitric-dev/membrane/pkg/plugins/secret" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/plugins/secret/secrets_manager/secrets_manager_test.go b/pkg/plugins/secret/secrets_manager/secrets_manager_test.go index 8cb748623..91092d158 100644 --- a/pkg/plugins/secret/secrets_manager/secrets_manager_test.go +++ b/pkg/plugins/secret/secrets_manager/secrets_manager_test.go @@ -21,7 +21,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" secretsmanager "github.com/aws/aws-sdk-go/service/secretsmanager" "github.com/golang/mock/gomock" - mocks "github.com/nitric-dev/membrane/mocks/mock_secrets_manager" + mocks "github.com/nitric-dev/membrane/mocks/secrets_manager" "github.com/nitric-dev/membrane/pkg/plugins/secret" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" From 231a9db7a663163235626ada74fc60c22684ff77 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 15:09:29 +1000 Subject: [PATCH 31/83] refactor: Separate document test suites out. --- tests/plugins/document/delete.go | 78 ++++ tests/plugins/document/get.go | 68 +++ tests/plugins/document/query.go | 556 ++++++++++++++++++++++ tests/plugins/document/set.go | 85 ++++ tests/plugins/document/suite.go | 765 +------------------------------ 5 files changed, 789 insertions(+), 763 deletions(-) create mode 100644 tests/plugins/document/delete.go create mode 100644 tests/plugins/document/get.go create mode 100644 tests/plugins/document/query.go create mode 100644 tests/plugins/document/set.go diff --git a/tests/plugins/document/delete.go b/tests/plugins/document/delete.go new file mode 100644 index 000000000..a2a97828f --- /dev/null +++ b/tests/plugins/document/delete.go @@ -0,0 +1,78 @@ +package document_suite + +import ( + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func DeleteTests(docPlugin document.DocumentService) { + Context("Delete", func() { + When("Blank key.Collection.Name", func() { + It("Should return error", func() { + key := document.Key{Id: "1"} + err := docPlugin.Delete(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Blank key.Id", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}} + err := docPlugin.Delete(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Delete", func() { + It("Should delete item successfully", func() { + docPlugin.Set(&UserKey1, UserItem1) + + err := docPlugin.Delete(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&UserKey1) + Expect(doc).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Sub Collection Delete", func() { + It("Should delete item successfully", func() { + docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) + + err := docPlugin.Delete(&Customer1.Orders[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&Customer1.Orders[0].Key) + Expect(doc).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Parent and Sub Collection Delete", func() { + It("Should delete all children", func() { + LoadCustomersData(docPlugin) + + col := document.Collection{ + Name: "orders", + Parent: &document.Key{ + Collection: &document.Collection{ + Name: "customers", + }, + }, + } + + result, err := docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) + Expect(err).To(BeNil()) + Expect(result.Documents).To(HaveLen(5)) + + err = docPlugin.Delete(&Customer1.Key) + Expect(err).ShouldNot(HaveOccurred()) + + err = docPlugin.Delete(&Customer2.Key) + Expect(err).ShouldNot(HaveOccurred()) + + result, err = docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) + Expect(err).To(BeNil()) + Expect(result.Documents).To(HaveLen(0)) + }) + }) + }) +} diff --git a/tests/plugins/document/get.go b/tests/plugins/document/get.go new file mode 100644 index 000000000..76f4bb34f --- /dev/null +++ b/tests/plugins/document/get.go @@ -0,0 +1,68 @@ +package document_suite + +import ( + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func GetTests(docPlugin document.DocumentService) { + Context("Get", func() { + When("Blank key.Collection.Name", func() { + It("Should return error", func() { + key := document.Key{Id: "1"} + _, err := docPlugin.Get(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Blank key.Id", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}} + _, err := docPlugin.Get(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Get", func() { + It("Should get item successfully", func() { + docPlugin.Set(&UserKey1, UserItem1) + + doc, err := docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Key).To(Equal(&UserKey1)) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) + }) + }) + When("Valid Sub Collection Get", func() { + It("Should store item successfully", func() { + docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) + + doc, err := docPlugin.Get(&Customer1.Orders[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Key).To(Equal(&Customer1.Orders[0].Key)) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) + }) + }) + When("Document Doesn't Exist", func() { + It("Should return NotFound error", func() { + key := document.Key{Collection: &document.Collection{Name: "items"}, Id: "not-exist"} + doc, err := docPlugin.Get(&key) + Expect(doc).To(BeNil()) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found")) + }) + }) + When("Valid Collection Get when there is a Sub Collection", func() { + It("Should store item successfully", func() { + docPlugin.Set(&Customer1.Key, Customer1.Content) + + doc, err := docPlugin.Get(&Customer1.Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Key).To(Equal(&Customer1.Key)) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Content)) + }) + }) + }) +} diff --git a/tests/plugins/document/query.go b/tests/plugins/document/query.go new file mode 100644 index 000000000..b580fc152 --- /dev/null +++ b/tests/plugins/document/query.go @@ -0,0 +1,556 @@ +package document_suite + +import ( + "fmt" + + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func QueryTests(docPlugin document.DocumentService) { + Context("Query", func() { + When("Invalid - blank key.Collection.Name", func() { + It("Should return an error", func() { + result, err := docPlugin.Query(&document.Collection{}, []document.QueryExpression{}, 0, nil) + Expect(result).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Invalid - nil expressions argument", func() { + It("Should return an error", func() { + result, err := docPlugin.Query(&document.Collection{Name: "users"}, nil, 0, nil) + Expect(result).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Empty database", func() { + It("Should return empty list", func() { + result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + Expect(result.PagingToken).To(BeNil()) + }) + }) + When("key: {users}, subcol: '', exp: []", func() { + It("Should return all users", func() { + LoadUsersData(docPlugin) + LoadCustomersData(docPlugin) + + result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Collection.Name).To(Equal("users")) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent).To(BeNil()) + } + }) + }) + When("key: {customers, nil}, subcol: '', exp: []", func() { + It("Should return 2 items", func() { + LoadCustomersData(docPlugin) + + result, err := docPlugin.Query(&CustomersColl, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer1.Content["email"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Key)) + Expect(result.PagingToken).To(BeNil()) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US]", func() { + It("Should return 1 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + } + result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer2.Key)) + Expect(result.PagingToken).To(BeNil()) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US, age > 40]", func() { + It("Should return 0 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + {Operand: "age", Operator: ">", Value: "40"}, + } + result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + }) + }) + When("key: {customers, key1}, subcol: orders", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[2].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*result.Documents[2].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[2].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number == 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number == 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number > 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number > 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number < 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number < 1]", func() { + It("Should return 0 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 5 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(5)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key {customers, nil}, subcol: orders, exps: [type startsWith scooter]", func() { + It("Should return 2 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[1].Content["testName"])) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[1].Key)) + }) + }) + When("key {customers, key1}, subcol: orders, exps: [type startsWith bike/road]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {items, nil}, subcol: '', exp: [], limit: 10", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + coll := document.Collection{ + Name: "items", + } + result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 10, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(10)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&coll, []document.QueryExpression{}, 10, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.PagingToken).To(BeNil()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + }) + }) + When("key: {items, nil}, subcol: '', exps: [letter > D], limit: 4", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + coll := document.Collection{ + Name: "items", + } + exps := []document.QueryExpression{ + {Operand: "letter", Operator: ">", Value: "D"}, + } + result, err := docPlugin.Query(&coll, exps, 4, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + + result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + Expect(result.PagingToken).To(BeEmpty()) + }) + }) + When("key: {parentItems, 1}, subcol: items, exp: [], limit: 10", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + result, err := docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(10)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.PagingToken).To(BeNil()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + }) + }) + When("key: {parentItems, 1}, subcol: items, exps: [letter > D], limit: 4", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "letter", Operator: ">", Value: "D"}, + } + result, err := docPlugin.Query(&ChildItemsCollection, exps, 4, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + + result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + Expect(result.PagingToken).To(BeEmpty()) + }) + }) + }) +} diff --git a/tests/plugins/document/set.go b/tests/plugins/document/set.go new file mode 100644 index 000000000..9c1a1a2b9 --- /dev/null +++ b/tests/plugins/document/set.go @@ -0,0 +1,85 @@ +package document_suite + +import ( + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func SetTests(docPlugin document.DocumentService) { + Context("Set", func() { + When("Blank key.Collection.Name", func() { + It("Should return error", func() { + key := document.Key{Id: "1"} + err := docPlugin.Set(&key, UserItem1) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Blank key.Id", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}} + err := docPlugin.Set(&key, UserItem1) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Nil item map", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}, Id: "1"} + err := docPlugin.Set(&key, nil) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid New Set", func() { + It("Should store new item successfully", func() { + err := docPlugin.Set(&UserKey1, UserItem1) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) + }) + }) + When("Valid Update Set", func() { + It("Should update existing item successfully", func() { + err := docPlugin.Set(&UserKey1, UserItem1) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) + + err = docPlugin.Set(&UserKey1, UserItem2) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err = docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem2["email"])) + }) + }) + When("Valid Sub Collection Set", func() { + It("Should store item successfully", func() { + err := docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&Customer1.Orders[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) + }) + }) + When("Valid Mutliple Sub Collection Set", func() { + It("Should store item successfully", func() { + err := docPlugin.Set(&Customer1.Reviews[0].Key, Customer1.Reviews[0].Content) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&Customer1.Reviews[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Reviews[0].Content)) + }) + }) + }) +} diff --git a/tests/plugins/document/suite.go b/tests/plugins/document/suite.go index 8ce718fbd..83208404e 100644 --- a/tests/plugins/document/suite.go +++ b/tests/plugins/document/suite.go @@ -12,14 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package document +package document_suite import ( - "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/document" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) // Simple 'users' collection test data @@ -169,7 +165,7 @@ var Customer1 = Customer{ }, Content: map[string]interface{}{ "title": "Good review", - "stars": "5", + "stars": "5", }, }, }, @@ -326,760 +322,3 @@ func LoadItemsData(docPlugin document.DocumentService) { } // Unit Test Functions -------------------------------------------------------- - -func GetTests(docPlugin document.DocumentService) { - Context("Get", func() { - When("Blank key.Collection.Name", func() { - It("Should return error", func() { - key := document.Key{Id: "1"} - _, err := docPlugin.Get(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Blank key.Id", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}} - _, err := docPlugin.Get(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Get", func() { - It("Should get item successfully", func() { - docPlugin.Set(&UserKey1, UserItem1) - - doc, err := docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Key).To(Equal(&UserKey1)) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) - }) - }) - When("Valid Sub Collection Get", func() { - It("Should store item successfully", func() { - docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) - - doc, err := docPlugin.Get(&Customer1.Orders[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Key).To(Equal(&Customer1.Orders[0].Key)) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) - }) - }) - When("Document Doesn't Exist", func() { - It("Should return NotFound error", func() { - key := document.Key{Collection: &document.Collection{Name: "items"}, Id: "not-exist"} - doc, err := docPlugin.Get(&key) - Expect(doc).To(BeNil()) - Expect(err).Should(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("not found")) - }) - }) - When("Valid Collection Get when there is a Sub Collection", func() { - It("Should store item successfully", func() { - docPlugin.Set(&Customer1.Key, Customer1.Content) - - doc, err := docPlugin.Get(&Customer1.Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Key).To(Equal(&Customer1.Key)) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Content)) - }) - }) - }) -} - -func SetTests(docPlugin document.DocumentService) { - Context("Set", func() { - When("Blank key.Collection.Name", func() { - It("Should return error", func() { - key := document.Key{Id: "1"} - err := docPlugin.Set(&key, UserItem1) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Blank key.Id", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}} - err := docPlugin.Set(&key, UserItem1) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Nil item map", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}, Id: "1"} - err := docPlugin.Set(&key, nil) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid New Set", func() { - It("Should store new item successfully", func() { - err := docPlugin.Set(&UserKey1, UserItem1) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) - }) - }) - When("Valid Update Set", func() { - It("Should update existing item successfully", func() { - err := docPlugin.Set(&UserKey1, UserItem1) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) - - err = docPlugin.Set(&UserKey1, UserItem2) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err = docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem2["email"])) - }) - }) - When("Valid Sub Collection Set", func() { - It("Should store item successfully", func() { - err := docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&Customer1.Orders[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) - }) - }) - When("Valid Mutliple Sub Collection Set", func() { - It("Should store item successfully", func() { - err := docPlugin.Set(&Customer1.Reviews[0].Key, Customer1.Reviews[0].Content) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&Customer1.Reviews[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Reviews[0].Content)) - }) - }) - }) -} - -func DeleteTests(docPlugin document.DocumentService) { - Context("Delete", func() { - When("Blank key.Collection.Name", func() { - It("Should return error", func() { - key := document.Key{Id: "1"} - err := docPlugin.Delete(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Blank key.Id", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}} - err := docPlugin.Delete(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Delete", func() { - It("Should delete item successfully", func() { - docPlugin.Set(&UserKey1, UserItem1) - - err := docPlugin.Delete(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&UserKey1) - Expect(doc).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Sub Collection Delete", func() { - It("Should delete item successfully", func() { - docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) - - err := docPlugin.Delete(&Customer1.Orders[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&Customer1.Orders[0].Key) - Expect(doc).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Parent and Sub Collection Delete", func() { - It("Should delete all children", func() { - LoadCustomersData(docPlugin) - - col := document.Collection{ - Name: "orders", - Parent: &document.Key{ - Collection: &document.Collection{ - Name: "customers", - }, - }, - } - - result, err := docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) - Expect(err).To(BeNil()) - Expect(result.Documents).To(HaveLen(5)) - - err = docPlugin.Delete(&Customer1.Key) - Expect(err).ShouldNot(HaveOccurred()) - - err = docPlugin.Delete(&Customer2.Key) - Expect(err).ShouldNot(HaveOccurred()) - - result, err = docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) - Expect(err).To(BeNil()) - Expect(result.Documents).To(HaveLen(0)) - }) - }) - }) -} - -func QueryTests(docPlugin document.DocumentService) { - Context("Query", func() { - When("Invalid - blank key.Collection.Name", func() { - It("Should return an error", func() { - result, err := docPlugin.Query(&document.Collection{}, []document.QueryExpression{}, 0, nil) - Expect(result).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Invalid - nil expressions argument", func() { - It("Should return an error", func() { - result, err := docPlugin.Query(&document.Collection{Name: "users"}, nil, 0, nil) - Expect(result).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Empty database", func() { - It("Should return empty list", func() { - result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - Expect(result.PagingToken).To(BeNil()) - }) - }) - When("key: {users}, subcol: '', exp: []", func() { - It("Should return all users", func() { - LoadUsersData(docPlugin) - LoadCustomersData(docPlugin) - - result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Collection.Name).To(Equal("users")) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent).To(BeNil()) - } - }) - }) - When("key: {customers, nil}, subcol: '', exp: []", func() { - It("Should return 2 items", func() { - LoadCustomersData(docPlugin) - - result, err := docPlugin.Query(&CustomersColl, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer1.Content["email"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Key)) - Expect(result.PagingToken).To(BeNil()) - }) - }) - When("key: {customers, nil}, subcol: '', exp: [country == US]", func() { - It("Should return 1 item", func() { - LoadCustomersData(docPlugin) - - exps := []document.QueryExpression{ - {Operand: "country", Operator: "==", Value: "US"}, - } - result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer2.Key)) - Expect(result.PagingToken).To(BeNil()) - }) - }) - When("key: {customers, nil}, subcol: '', exp: [country == US, age > 40]", func() { - It("Should return 0 item", func() { - LoadCustomersData(docPlugin) - - exps := []document.QueryExpression{ - {Operand: "country", Operator: "==", Value: "US"}, - {Operand: "age", Operator: ">", Value: "40"}, - } - result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - }) - }) - When("key: {customers, key1}, subcol: orders", func() { - It("Should return 3 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[2].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(*result.Documents[2].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[2].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number == 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "==", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number == 1]", func() { - It("Should return 1 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "==", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number > 1]", func() { - It("Should return 3 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number > 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number < 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number < 1]", func() { - It("Should return 0 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number >= 1]", func() { - It("Should return 5 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(5)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number >= 1]", func() { - It("Should return 3 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number <= 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number <= 1]", func() { - It("Should return 1 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key {customers, nil}, subcol: orders, exps: [type startsWith scooter]", func() { - It("Should return 2 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "type", Operator: "startsWith", Value: "scooter"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[1].Content["testName"])) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[1].Key)) - }) - }) - When("key {customers, key1}, subcol: orders, exps: [type startsWith bike/road]", func() { - It("Should return 1 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "type", Operator: "startsWith", Value: "scooter"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {items, nil}, subcol: '', exp: [], limit: 10", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - coll := document.Collection{ - Name: "items", - } - result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 10, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(10)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&coll, []document.QueryExpression{}, 10, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.PagingToken).To(BeNil()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - }) - }) - When("key: {items, nil}, subcol: '', exps: [letter > D], limit: 4", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - coll := document.Collection{ - Name: "items", - } - exps := []document.QueryExpression{ - {Operand: "letter", Operator: ">", Value: "D"}, - } - result, err := docPlugin.Query(&coll, exps, 4, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - - result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - Expect(result.PagingToken).To(BeEmpty()) - }) - }) - When("key: {parentItems, 1}, subcol: items, exp: [], limit: 10", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - result, err := docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(10)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.PagingToken).To(BeNil()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - }) - }) - When("key: {parentItems, 1}, subcol: items, exps: [letter > D], limit: 4", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - exps := []document.QueryExpression{ - {Operand: "letter", Operator: ">", Value: "D"}, - } - result, err := docPlugin.Query(&ChildItemsCollection, exps, 4, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - - result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - Expect(result.PagingToken).To(BeEmpty()) - }) - }) - }) -} From b13e262b8584175efc8f3cbe729f08d8b352beda Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 15:09:45 +1000 Subject: [PATCH 32/83] test: Add document streaming test harness. --- tests/plugins/document/stream.go | 446 +++++++++++++++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 tests/plugins/document/stream.go diff --git a/tests/plugins/document/stream.go b/tests/plugins/document/stream.go new file mode 100644 index 000000000..05120b145 --- /dev/null +++ b/tests/plugins/document/stream.go @@ -0,0 +1,446 @@ +package document_suite + +import ( + "io" + + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func unwrapIter(iter document.DocumentIterator) []*document.Document { + docs := make([]*document.Document, 0) + for { + d, err := iter() + + if err != nil { + Expect(err).To(Equal(io.EOF)) + break + } + + docs = append(docs, d) + } + + return docs +} + +func QueryStreamTests(docPlugin document.DocumentService) { + Context("QueryStream", func() { + // Validation Tests + When("Invalid - blank key.Collection.Name", func() { + It("Should return an iterator that errors", func() { + iter := docPlugin.QueryStream(&document.Collection{}, []document.QueryExpression{}, 0) + Expect(iter).ToNot(BeNil()) + + _, err := iter() + Expect(err).Should(HaveOccurred()) + Expect(err).ToNot(Equal(io.EOF)) + }) + }) + When("Invalid - nil expressions argument", func() { + It("Should return an iterator that errors", func() { + iter := docPlugin.QueryStream(&document.Collection{Name: "users"}, nil, 0) + Expect(iter).ToNot(BeNil()) + + _, err := iter() + Expect(err).Should(HaveOccurred()) + Expect(err).ToNot(Equal(io.EOF)) + }) + }) + When("Empty database", func() { + It("Should return io.EOF", func() { + iter := docPlugin.QueryStream(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0) + + Expect(iter).ToNot(BeNil()) + + doc, err := iter() + Expect(doc).To(BeNil()) + Expect(err).To(HaveOccurred()) + Expect(err).To(Equal(io.EOF)) + }) + }) + + // Query Tests + When("key: {users}, subcol: '', exp: []", func() { + It("Should return all users", func() { + LoadUsersData(docPlugin) + LoadCustomersData(docPlugin) + + iter := docPlugin.QueryStream(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0) + + for { + d, err := iter() + + if err != nil { + Expect(err).To(Equal(io.EOF)) + break + } + + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Collection.Name).To(Equal("users")) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent).To(BeNil()) + } + }) + }) + When("key: {customers, nil}, subcol: '', exp: []", func() { + It("Should return 2 items", func() { + LoadCustomersData(docPlugin) + + iter := docPlugin.QueryStream(&CustomersColl, []document.QueryExpression{}, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["email"]).To(BeEquivalentTo(Customer1.Content["email"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US]", func() { + It("Should return 1 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + } + + iter := docPlugin.QueryStream(&CustomersColl, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US, age > 40]", func() { + It("Should return 0 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + {Operand: "age", Operator: ">", Value: "40"}, + } + + iter := docPlugin.QueryStream(&CustomersColl, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(0)) + }) + }) + When("key: {customers, key1}, subcol: orders", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + + iter := docPlugin.QueryStream(&coll, []document.QueryExpression{}, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(3)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[2].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*docs[2].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[2].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number == 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number == 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number > 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(3)) + + for _, d := range docs { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number > 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number < 1]", func() { + It("Should return 0 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(0)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number < 1]", func() { + It("Should return 0 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(0)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 5 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(5)) + + for _, d := range docs { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(3)) + + for _, d := range docs { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key {customers, nil}, subcol: orders, exps: [type startsWith scooter]", func() { + It("Should return 2 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[1].Content["testName"])) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Orders[1].Key)) + }) + }) + When("key {customers, key1}, subcol: orders, exps: [type startsWith bike/road]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {items, nil}, subcol: '', exp: [], limit: 10", func() { + It("Should return limited results", func() { + LoadItemsData(docPlugin) + + coll := document.Collection{ + Name: "items", + } + + iter := docPlugin.QueryStream(&coll, []document.QueryExpression{}, 10) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(10)) + }) + }) + }) +} From 16de05b652499755963c45fbc8b6244b56a8f4fa Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 15:12:21 +1000 Subject: [PATCH 33/83] chore: add missing copyright headers. --- tests/plugins/document/delete.go | 14 ++++++++++++++ tests/plugins/document/get.go | 14 ++++++++++++++ tests/plugins/document/query.go | 14 ++++++++++++++ tests/plugins/document/set.go | 14 ++++++++++++++ tests/plugins/document/stream.go | 14 ++++++++++++++ 5 files changed, 70 insertions(+) diff --git a/tests/plugins/document/delete.go b/tests/plugins/document/delete.go index a2a97828f..57d8d5725 100644 --- a/tests/plugins/document/delete.go +++ b/tests/plugins/document/delete.go @@ -1,3 +1,17 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package document_suite import ( diff --git a/tests/plugins/document/get.go b/tests/plugins/document/get.go index 76f4bb34f..d915158d8 100644 --- a/tests/plugins/document/get.go +++ b/tests/plugins/document/get.go @@ -1,3 +1,17 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package document_suite import ( diff --git a/tests/plugins/document/query.go b/tests/plugins/document/query.go index b580fc152..450b5edb9 100644 --- a/tests/plugins/document/query.go +++ b/tests/plugins/document/query.go @@ -1,3 +1,17 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package document_suite import ( diff --git a/tests/plugins/document/set.go b/tests/plugins/document/set.go index 9c1a1a2b9..1ad3f3fec 100644 --- a/tests/plugins/document/set.go +++ b/tests/plugins/document/set.go @@ -1,3 +1,17 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package document_suite import ( diff --git a/tests/plugins/document/stream.go b/tests/plugins/document/stream.go index 05120b145..32442bc58 100644 --- a/tests/plugins/document/stream.go +++ b/tests/plugins/document/stream.go @@ -1,3 +1,17 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package document_suite import ( From a2ea076175c20fc56be293d86f15dfebfc01a589 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 15:30:13 +1000 Subject: [PATCH 34/83] chore: temporarily remove empty test. --- tests/plugins/document/stream.go | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/tests/plugins/document/stream.go b/tests/plugins/document/stream.go index 32442bc58..2a48d7819 100644 --- a/tests/plugins/document/stream.go +++ b/tests/plugins/document/stream.go @@ -61,18 +61,6 @@ func QueryStreamTests(docPlugin document.DocumentService) { Expect(err).ToNot(Equal(io.EOF)) }) }) - When("Empty database", func() { - It("Should return io.EOF", func() { - iter := docPlugin.QueryStream(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0) - - Expect(iter).ToNot(BeNil()) - - doc, err := iter() - Expect(doc).To(BeNil()) - Expect(err).To(HaveOccurred()) - Expect(err).To(Equal(io.EOF)) - }) - }) // Query Tests When("key: {users}, subcol: '', exp: []", func() { @@ -82,14 +70,11 @@ func QueryStreamTests(docPlugin document.DocumentService) { iter := docPlugin.QueryStream(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0) - for { - d, err := iter() + docs := unwrapIter(iter) - if err != nil { - Expect(err).To(Equal(io.EOF)) - break - } + Expect(docs).To(HaveLen(3)) + for _, d := range docs { Expect(d.Key).ToNot(BeNil()) Expect(d.Key.Collection.Name).To(Equal("users")) Expect(d.Key.Id).ToNot(Equal("")) From 74ddeabebc02df07c7c1cc62d1baffc963ec2993 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Wed, 15 Sep 2021 09:20:43 +1000 Subject: [PATCH 35/83] feat(documents/boltdb): Add boltdb streaming. --- pkg/plugins/document/boltdb/boltdb.go | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/pkg/plugins/document/boltdb/boltdb.go b/pkg/plugins/document/boltdb/boltdb.go index 3dce599de..4154b9fd3 100644 --- a/pkg/plugins/document/boltdb/boltdb.go +++ b/pkg/plugins/document/boltdb/boltdb.go @@ -16,6 +16,7 @@ package boltdb_service import ( "fmt" + "io" "os" "path/filepath" "strconv" @@ -359,6 +360,61 @@ func (s *BoltDocService) Query(collection *document.Collection, expressions []do }, nil } +func (s *BoltDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { + newErr := errors.ErrorsWithScope( + "BoltDocService.Query", + map[string]interface{}{ + "collection": collection, + }, + ) + + return s.query(collection, expressions, limit, pagingToken, newErr) +} + +func (s *BoltDocService) Stream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "BoltDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) + + var tmpLimit = limit + res, fetchErr := s.query(collection, expressions, limit, nil, newErr) + + // Initial fetch + var documents []document.Document = res.Documents + var pagingToken = res.PagingToken + + return func() (*document.Document, error) { + // check the iteration state + if tmpLimit == 0 { + // we've reached the limit of reading + return nil, io.EOF + } else if pagingToken != nil && len(documents) == 0 { + // we've run out of documents and have more pages to read + res, fetchErr = s.query(collection, expressions, tmpLimit, pagingToken, newErr) + documents = res.Documents + pagingToken = res.PagingToken + } else if pagingToken == nil && len(documents) == 0 { + // we're all out of documents and pages before hitting the limit + return nil, io.EOF + } + + // We received an error fetching the docs + if fetchErr != nil { + return nil, fetchErr + } + + // pop the first element + var doc document.Document + doc, documents = documents[0], documents[1:] + tmpLimit = tmpLimit - 1 + + return &doc, nil + } +} + // New - Create a new dev KV plugin func New() (*BoltDocService, error) { dbDir := utils.GetEnv("LOCAL_DB_DIR", utils.GetRelativeDevPath(DEV_SUB_DIRECTORY)) From ba39c60a747379f7bcbc684f9812792eca3418a5 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Wed, 15 Sep 2021 09:24:39 +1000 Subject: [PATCH 36/83] chore: Update errors and address conflicts. --- pkg/plugins/document/boltdb/boltdb.go | 9 +-------- pkg/plugins/errors/plugin_error.go | 4 +++- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/plugins/document/boltdb/boltdb.go b/pkg/plugins/document/boltdb/boltdb.go index 4154b9fd3..1b20636c9 100644 --- a/pkg/plugins/document/boltdb/boltdb.go +++ b/pkg/plugins/document/boltdb/boltdb.go @@ -216,14 +216,7 @@ func (s *BoltDocService) Delete(key *document.Key) error { return nil } -func (s *BoltDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { - newErr := errors.ErrorsWithScope( - "BoltDocService.Query", - map[string]interface{}{ - "collection": collection, - }, - ) - +func (s *BoltDocService) query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string, newErr errors.ErrorFactory) (*document.QueryResult, error) { if err := document.ValidateQueryCollection(collection); err != nil { return nil, newErr( codes.InvalidArgument, diff --git a/pkg/plugins/errors/plugin_error.go b/pkg/plugins/errors/plugin_error.go index 5cb5ace42..71d7d2f09 100644 --- a/pkg/plugins/errors/plugin_error.go +++ b/pkg/plugins/errors/plugin_error.go @@ -50,8 +50,10 @@ func Code(e error) codes.Code { return codes.Unknown } +type ErrorFactory = func(c codes.Code, msg string, cause error) error + // ErrorsWithScope - Returns a new reusable error factory with the given scope -func ErrorsWithScope(scope string, args map[string]interface{}) func(c codes.Code, msg string, cause error) error { +func ErrorsWithScope(scope string, args map[string]interface{}) ErrorFactory { return func(code codes.Code, msg string, cause error) error { return &PluginError{ Code: code, From 0b3844a0121d9ae8f6d550d4d2c89bae005c0a45 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Thu, 16 Sep 2021 11:28:56 +1000 Subject: [PATCH 37/83] chore: fix plugin streaming method name. --- pkg/plugins/document/boltdb/boltdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugins/document/boltdb/boltdb.go b/pkg/plugins/document/boltdb/boltdb.go index 1b20636c9..a60f64825 100644 --- a/pkg/plugins/document/boltdb/boltdb.go +++ b/pkg/plugins/document/boltdb/boltdb.go @@ -364,7 +364,7 @@ func (s *BoltDocService) Query(collection *document.Collection, expressions []do return s.query(collection, expressions, limit, pagingToken, newErr) } -func (s *BoltDocService) Stream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { +func (s *BoltDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { newErr := errors.ErrorsWithScope( "BoltDocService.QueryStream", map[string]interface{}{ From eb5b0424e8f78b60132eaaede020c59abd85e4fc Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 13:44:09 +1000 Subject: [PATCH 38/83] chore: avoid panic by returning error only iterator on initial fetch. --- pkg/plugins/document/boltdb/boltdb.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/plugins/document/boltdb/boltdb.go b/pkg/plugins/document/boltdb/boltdb.go index a60f64825..78d1aac66 100644 --- a/pkg/plugins/document/boltdb/boltdb.go +++ b/pkg/plugins/document/boltdb/boltdb.go @@ -373,11 +373,21 @@ func (s *BoltDocService) QueryStream(collection *document.Collection, expression ) var tmpLimit = limit - res, fetchErr := s.query(collection, expressions, limit, nil, newErr) + var documents []document.Document + var pagingToken map[string]string // Initial fetch - var documents []document.Document = res.Documents - var pagingToken = res.PagingToken + res, fetchErr := s.query(collection, expressions, limit, nil, newErr) + + if fetchErr != nil { + // Return an error only iterator if the initial fetch failed + return func() (*document.Document, error) { + return nil, fetchErr + } + } + + documents = res.Documents + pagingToken = res.PagingToken return func() (*document.Document, error) { // check the iteration state From 5716fb6fc63397a4c9a5ca981525c41630ff4121 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 15:29:09 +1000 Subject: [PATCH 39/83] chore: Fix limit bug always returning empty. --- pkg/plugins/document/boltdb/boltdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugins/document/boltdb/boltdb.go b/pkg/plugins/document/boltdb/boltdb.go index 78d1aac66..d4423495f 100644 --- a/pkg/plugins/document/boltdb/boltdb.go +++ b/pkg/plugins/document/boltdb/boltdb.go @@ -391,7 +391,7 @@ func (s *BoltDocService) QueryStream(collection *document.Collection, expression return func() (*document.Document, error) { // check the iteration state - if tmpLimit == 0 { + if tmpLimit == 0 && limit > 0 { // we've reached the limit of reading return nil, io.EOF } else if pagingToken != nil && len(documents) == 0 { From 5eafa4fd9f8789de8001508ea5d98d0888317942 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 15:31:31 +1000 Subject: [PATCH 40/83] test: Connect document streaming test suite. --- tests/plugins/document/boltdb/boltdb_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/plugins/document/boltdb/boltdb_test.go b/tests/plugins/document/boltdb/boltdb_test.go index bf8eb2cd1..06a7629d9 100644 --- a/tests/plugins/document/boltdb/boltdb_test.go +++ b/tests/plugins/document/boltdb/boltdb_test.go @@ -48,4 +48,5 @@ var _ = Describe("Bolt", func() { test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) From ad1e754b6a7ab1e69a023778e0d89d09d25f6c52 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 14:22:27 +1000 Subject: [PATCH 41/83] feat(documents/firestore): Add streaming support to the firestore plugin. --- pkg/plugins/document/firestore/firestore.go | 161 +++++++++++++------- 1 file changed, 110 insertions(+), 51 deletions(-) diff --git a/pkg/plugins/document/firestore/firestore.go b/pkg/plugins/document/firestore/firestore.go index e38d06a27..aa5344842 100644 --- a/pkg/plugins/document/firestore/firestore.go +++ b/pkg/plugins/document/firestore/firestore.go @@ -17,6 +17,7 @@ package firestore_service import ( "context" "fmt" + "io" "strings" "github.com/nitric-dev/membrane/pkg/plugins/document" @@ -185,6 +186,35 @@ func (s *FirestoreDocService) Delete(key *document.Key) error { return nil } +// +func (s *FirestoreDocService) buildQuery(collection *document.Collection, expressions []document.QueryExpression, limit int) (query firestore.Query, orderBy string) { + // Select correct root collection to perform query on + query = s.getQueryRoot(collection) + + for _, exp := range expressions { + expOperand := exp.Operand + if exp.Operator == "startsWith" { + expVal := fmt.Sprintf("%v", exp.Value) + endRangeValue := document.GetEndRangeValue(expVal) + query = query.Where(expOperand, ">=", exp.Value).Where(expOperand, "<", endRangeValue) + + } else { + query = query.Where(expOperand, exp.Operator, exp.Value) + } + + if exp.Operator != "==" && limit > 0 && orderBy == "" { + query = query.OrderBy(expOperand, firestore.Asc) + orderBy = expOperand + } + } + + if limit > 0 { + query = query.Limit(limit) + } + + return +} + func (s *FirestoreDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "FirestoreDocService.Query", @@ -214,41 +244,17 @@ func (s *FirestoreDocService) Query(collection *document.Collection, expressions } // Select correct root collection to perform query on - query := s.getQueryRoot(collection) - - var orderByAttrib string - - for _, exp := range expressions { - expOperand := exp.Operand - if exp.Operator == "startsWith" { - expVal := fmt.Sprintf("%v", exp.Value) - endRangeValue := document.GetEndRangeValue(expVal) - query = query.Where(expOperand, ">=", exp.Value).Where(expOperand, "<", endRangeValue) - - } else { - query = query.Where(expOperand, exp.Operator, exp.Value) - } - - if exp.Operator != "==" && limit > 0 && orderByAttrib == "" { - query = query.OrderBy(expOperand, firestore.Asc) - orderByAttrib = expOperand - } - } - - if limit > 0 { - query = query.Limit(limit) - - if len(pagingToken) > 0 { - query = query.OrderBy(firestore.DocumentID, firestore.Asc) + query, orderBy := s.buildQuery(collection, expressions, limit) - if tokens, ok := pagingToken[pagingTokens]; ok { - var vals []interface{} - for _, v := range strings.Split(tokens, "|") { - vals = append(vals, v) - } - query = query.StartAfter(vals...) + if len(pagingToken) > 0 { + query = query.OrderBy(firestore.DocumentID, firestore.Asc) + if tokens, ok := pagingToken[pagingTokens]; ok { + var vals []interface{} + for _, v := range strings.Split(tokens, "|") { + vals = append(vals, v) } + query = query.StartAfter(vals...) } } @@ -261,31 +267,15 @@ func (s *FirestoreDocService) Query(collection *document.Collection, expressions err, ) } - sdkDoc := document.Document{ - Content: docSnp.Data(), - Key: &document.Key{ - Collection: collection, - Id: docSnp.Ref.ID, - }, - } - - if p := docSnp.Ref.Parent.Parent; p != nil { - sdkDoc.Key.Collection = &document.Collection{ - Name: collection.Name, - Parent: &document.Key{ - Collection: collection.Parent.Collection, - Id: p.ID, - }, - } - } + sdkDoc := docSnpToDocument(collection, docSnp) queryResult.Documents = append(queryResult.Documents, sdkDoc) // If query limit configured determine continue tokens if limit > 0 && len(queryResult.Documents) == limit { tokens := "" - if orderByAttrib != "" { - tokens = fmt.Sprintf("%v", docSnp.Data()[orderByAttrib]) + "|" + if orderBy != "" { + tokens = fmt.Sprintf("%v", docSnp.Data()[orderBy]) + "|" } tokens += docSnp.Ref.ID @@ -298,6 +288,75 @@ func (s *FirestoreDocService) Query(collection *document.Collection, expressions return queryResult, nil } +func (s *FirestoreDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "FirestoreDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) + + colErr := document.ValidateQueryCollection(collection) + expErr := document.ValidateExpressions(expressions) + + if colErr != nil || expErr != nil { + // Return an error only iterator + return func() (*document.Document, error) { + return nil, newErr( + codes.InvalidArgument, + "invalid arguments", + fmt.Errorf("collection error:%v, expression error: %v", colErr, expErr), + ) + } + } + + query, _ := s.buildQuery(collection, expressions, limit) + + iter := query.Documents(s.context) + + return func() (*document.Document, error) { + docSnp, err := iter.Next() + + if err != nil { + if err == iterator.Done { + return nil, io.EOF + } + + return nil, newErr( + codes.Internal, + "error querying value", + err, + ) + } + + sdkDoc := docSnpToDocument(collection, docSnp) + + return &sdkDoc, nil + } +} + +func docSnpToDocument(col *document.Collection, snp *firestore.DocumentSnapshot) document.Document { + sdkDoc := document.Document{ + Content: snp.Data(), + Key: &document.Key{ + Collection: col, + Id: snp.Ref.ID, + }, + } + + if p := snp.Ref.Parent.Parent; p != nil { + sdkDoc.Key.Collection = &document.Collection{ + Name: col.Name, + Parent: &document.Key{ + Collection: col.Parent.Collection, + Id: p.ID, + }, + } + } + + return sdkDoc +} + func New() (document.DocumentService, error) { ctx := context.Background() From bcb3316acf88ad3db7179dc10bd6c68a81cee8db Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 15:32:31 +1000 Subject: [PATCH 42/83] test(documents/firestore): Connect streaming test harness --- tests/plugins/document/firestore/firestore_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/plugins/document/firestore/firestore_test.go b/tests/plugins/document/firestore/firestore_test.go index bfb863316..8904a7f65 100644 --- a/tests/plugins/document/firestore/firestore_test.go +++ b/tests/plugins/document/firestore/firestore_test.go @@ -75,4 +75,5 @@ var _ = Describe("Firestore", func() { test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) From 272cb42d61456288ceef7a66e431b36da30be072 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Mon, 23 Aug 2021 15:58:47 +1000 Subject: [PATCH 43/83] feat: Initial event grid plugin --- pkg/plugins/events/eventgrid/eventgrid.go | 173 ++++++++++++++++ .../events/eventgrid/eventgrid_suite_test.go | 27 +++ .../events/eventgrid/eventgrid_test.go | 184 ++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 pkg/plugins/events/eventgrid/eventgrid.go create mode 100644 pkg/plugins/events/eventgrid/eventgrid_suite_test.go create mode 100644 pkg/plugins/events/eventgrid/eventgrid_test.go diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go new file mode 100644 index 000000000..91bc659eb --- /dev/null +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -0,0 +1,173 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventgrid_service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid" + "github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi" + eventgridmgmt "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid" + eventgridmgmtapi "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi" + "github.com/Azure/go-autorest/autorest/date" + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" + "github.com/nitric-dev/membrane/pkg/plugins/events" + "github.com/nitric-dev/membrane/pkg/utils" +) + +type EventGridEventService struct { + events.UnimplementedeventsPlugin + client eventgridapi.BaseClientAPI + topicClient eventgridmgmtapi.TopicsClientAPI + topicLocation string +} + +func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.NitricEvent) ([]eventgrid.Event, error) { + payload, err := json.Marshal(event.Payload) + if err != nil { + return nil, err + } + subject := fmt.Sprintf("Subject/%s", topic) + eventType := fmt.Sprintf("Type/%s", topic) + azureEvent := []eventgrid.Event{ + { + ID: &event.ID, + Data: &payload, + Topic: &topic, + EventType: &eventType, + Subject: &subject, + EventTime: &date.Time{time.Now()}, + }, + } + + return azureEvent, nil +} + +func (s *EventGridEventService) ListTopics() ([]string, error) { + newErr := errors.ErrorsWithScope("EventGridEventService.ListTopics") + ctx := context.TODO() + pageLength := int32(20) + + results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) + + if err != nil { + return nil, newErr( + codes.Internal, + "azure list topics error", + err, + ) + } + + var topics []string + + for results.NotDone() { + topicsList := results.Values() + for _, topic := range topicsList { + topics = append(topics, *topic.Name) + } + results.NextWithContext(ctx) + } + + return topics, nil +} + +func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) error { + newErr := errors.ErrorsWithScope("EventGridEventService.Publish") + + if len(topic) == 0 { + return newErr( + codes.InvalidArgument, + "provide non-blank topic", + fmt.Errorf("provided invalid topic"), + ) + } + if event == nil { + return newErr( + codes.InvalidArgument, + "provide non-nil event", + fmt.Errorf("provided invalid event"), + ) + } + ctx := context.TODO() + + //Convert topic -> topic1.westus2-1.eventgrid.azure.net + topicHostName := fmt.Sprintf("%s.%s.eventgrid.azure.net", topic, strings.ToLower(s.topicLocation)) + + events, err := s.NitricEventToEvent(topic, event) + if err != nil { + return err + } + + result, err := s.client.PublishEvents(ctx, topicHostName, events) + + if err != nil { + return newErr( + codes.Internal, + "azure publish event error", + err, + ) + } + + if result.StatusCode != 200 { + return newErr( + codes.Internal, + "azure publish event returned non-200 status code", + fmt.Errorf(string(rune(result.StatusCode))), + ) + } + return nil +} + +func New() (events.EventService, error) { + newErr := errors.ErrorsWithScope("EventGridEventService.New") + topicLocation := utils.GetEnv("AZURE_TOPIC_LOCATION", "") + subscriptionID := utils.GetEnv("AZURE_SUBSCRIPTION_ID", "") + + if len(topicLocation) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_TOPIC_LOCATION not configured", + fmt.Errorf(""), + ) + } + if len(subscriptionID) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_SUBSCRIPTION_ID not configured", + fmt.Errorf(""), + ) + } + client := eventgrid.New() + topicClient := eventgridmgmt.NewTopicsClient(subscriptionID) + + return &EventGridEventService{ + client: client, + topicClient: topicClient, + topicLocation: topicLocation, + }, nil +} + +func NewWithClient(client eventgridapi.BaseClientAPI, topicClient eventgridmgmtapi.TopicsClientAPI) (events.EventService, error) { + return &EventGridEventService{ + client: client, + topicClient: topicClient, + topicLocation: "local1-test", + }, nil +} diff --git a/pkg/plugins/events/eventgrid/eventgrid_suite_test.go b/pkg/plugins/events/eventgrid/eventgrid_suite_test.go new file mode 100644 index 000000000..08b9902ea --- /dev/null +++ b/pkg/plugins/events/eventgrid/eventgrid_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventgrid_service_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPubsub(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Event Grid Event Service Suite") +} diff --git a/pkg/plugins/events/eventgrid/eventgrid_test.go b/pkg/plugins/events/eventgrid/eventgrid_test.go new file mode 100644 index 000000000..e667d0d4d --- /dev/null +++ b/pkg/plugins/events/eventgrid/eventgrid_test.go @@ -0,0 +1,184 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventgrid_service_test + +import ( + "context" + "fmt" + "net/http" + + eventgridmgmt "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid" + "github.com/Azure/go-autorest/autorest" + "github.com/golang/mock/gomock" + mock_eventgrid "github.com/nitric-dev/membrane/mocks/mock_event_grid" + "github.com/nitric-dev/membrane/pkg/plugins/events" + eventgrid_service "github.com/nitric-dev/membrane/pkg/plugins/events/eventgrid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Event Grid Plugin", func() { + When("Listing Available Topics", func() { + When("There are no topics available", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + Expect(topicClient).NotTo(BeNil()) + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(eventgridmgmt.TopicsListResultPage{}, nil).Times(1) + + It("Should return an empty list of topics", func() { + topics, err := eventgridPlugin.ListTopics() + Expect(err).To(BeNil()) + Expect(topics).To(BeEmpty()) + }) + }) + + When("There are topics available", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + topicName := "Test" + topicListResponsePage := eventgridmgmt.NewTopicsListResultPage( + eventgridmgmt.TopicsListResult{ + Value: &[]eventgridmgmt.Topic{ + { + Name: &topicName, + }, + }, + }, + func(context.Context, eventgridmgmt.TopicsListResult) (eventgridmgmt.TopicsListResult, error) { + return eventgridmgmt.TopicsListResult{}, nil + }, + ) + + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(topicListResponsePage, nil).Times(1) + + It("Should return all available topics", func() { + topics, err := eventgridPlugin.ListTopics() + Expect(err).To(BeNil()) + Expect(topics).To(ContainElement("Test")) + }) + }) + }) + + When("Publishing Messages", func() { + event := &events.NitricEvent{ + ID: "Test", + PayloadType: "Test", + Payload: map[string]interface{}{ + "Test": "Test", + }, + } + + When("To a topic that does not exist", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + eventgridClient.EXPECT().PublishEvents( + gomock.Any(), + "Test.local1-test.eventgrid.azure.net", + gomock.Any(), + ).Return(autorest.Response{}, fmt.Errorf("Topic does not exist")).Times(1) + + It("should return an error", func() { + err := eventgridPlugin.Publish("Test", event) + Expect(err).Should(HaveOccurred()) + }) + }) + + When("To a topic that is unauthorised", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + eventgridClient.EXPECT().PublishEvents( + gomock.Any(), + "Test.local1-test.eventgrid.azure.net", + gomock.Any(), + ).Return(autorest.Response{ + &http.Response{ + StatusCode: 404, + }, + }, nil).Times(1) + + It("should return an error", func() { + err := eventgridPlugin.Publish("Test", event) + Expect(err).Should(HaveOccurred()) + }) + }) + + When("To a topic that does exist", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + eventgridClient.EXPECT().PublishEvents( + gomock.Any(), + "Test.local1-test.eventgrid.azure.net", + gomock.Any(), + ).Return(autorest.Response{ + &http.Response{ + StatusCode: 200, + }, + }, nil).Times(1) + + It("should successfully publish the message", func() { + err := eventgridPlugin.Publish("Test", event) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + When("Providing an empty topic", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + It("should return an error", func() { + err := eventgridPlugin.Publish("", event) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("provide non-blank topic")) + }) + }) + + When("Providing an empty event", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + It("should return an error", func() { + err := eventgridPlugin.Publish("Test", nil) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("provide non-nil event")) + }) + }) + }) +}) From 154aec9d9ba62393030aeb398f37dda66b5df608 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 24 Aug 2021 13:21:57 +1000 Subject: [PATCH 44/83] feat: Add access token for event grid requests --- pkg/plugins/events/eventgrid/eventgrid.go | 78 ++++++++++++++++++++++- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go index 91bc659eb..cd9217962 100644 --- a/pkg/plugins/events/eventgrid/eventgrid.go +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -18,6 +18,8 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "net/url" "strings" "time" @@ -37,6 +39,37 @@ type EventGridEventService struct { client eventgridapi.BaseClientAPI topicClient eventgridmgmtapi.TopicsClientAPI topicLocation string + accessToken AzureAccessToken +} + +type AzureAccessToken struct { + TokenType string `json:"token_type"` + ExpiresIn string `json:"expires_in"` + ExtExpiresIn string `json:"ext_expires_in"` + ExpiresOn string `json:"expires_on"` + NotBefore string `json:"not_before"` + Resource string `json:"resource"` + AccessToken string `json:"access_token"` +} + +func GetToken(tenantId string, clientId string, clientSecret string) (AzureAccessToken, error) { + requestAccessTokenUri := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/token", tenantId) + requestBody := url.Values{ + "grant_type": {"client_credentials"}, + "client_id": {clientId}, + "client_secret": {clientSecret}, + "resource": {"https://management.azure.com/"}, + } + resp, err := http.PostForm(requestAccessTokenUri, requestBody) + if err != nil { + return AzureAccessToken{}, err + } + + var result AzureAccessToken + + json.NewDecoder(resp.Body).Decode(&result) + + return result, nil } func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.NitricEvent) ([]eventgrid.Event, error) { @@ -62,7 +95,11 @@ func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.N func (s *EventGridEventService) ListTopics() ([]string, error) { newErr := errors.ErrorsWithScope("EventGridEventService.ListTopics") - ctx := context.TODO() + ctx := context.Background() + ctx.Value(map[string]string{ + "Authorization": fmt.Sprintf("%s %s", s.accessToken.TokenType, s.accessToken.AccessToken), + }) + pageLength := int32(20) results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) @@ -90,6 +127,10 @@ func (s *EventGridEventService) ListTopics() ([]string, error) { func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope("EventGridEventService.Publish") + ctx := context.Background() + ctx.Value(map[string]string{ + "Authorization": fmt.Sprintf("%s %s", s.accessToken.TokenType, s.accessToken.AccessToken), + }) if len(topic) == 0 { return newErr( @@ -105,7 +146,6 @@ func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) fmt.Errorf("provided invalid event"), ) } - ctx := context.TODO() //Convert topic -> topic1.westus2-1.eventgrid.azure.net topicHostName := fmt.Sprintf("%s.%s.eventgrid.azure.net", topic, strings.ToLower(s.topicLocation)) @@ -139,7 +179,31 @@ func New() (events.EventService, error) { newErr := errors.ErrorsWithScope("EventGridEventService.New") topicLocation := utils.GetEnv("AZURE_TOPIC_LOCATION", "") subscriptionID := utils.GetEnv("AZURE_SUBSCRIPTION_ID", "") + tenantId := utils.GetEnv("AZURE_TENANT_ID", "") + clientId := utils.GetEnv("AZURE_CLIENT_ID", "") + clientSecret := utils.GetEnv("AZURE_CLIENT_SECRET", "") + if len(tenantId) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_TENANT_ID not configured", + fmt.Errorf(""), + ) + } + if len(clientId) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_CLIENT_ID not configured", + fmt.Errorf(""), + ) + } + if len(clientSecret) == 0 { + return nil, newErr( + codes.InvalidArgument, + "AZURE_CLIENT_SECRET not configured", + fmt.Errorf(""), + ) + } if len(topicLocation) == 0 { return nil, newErr( codes.InvalidArgument, @@ -156,11 +220,19 @@ func New() (events.EventService, error) { } client := eventgrid.New() topicClient := eventgridmgmt.NewTopicsClient(subscriptionID) - + accessToken, err := GetToken(tenantId, clientId, clientSecret) + if err != nil { + return nil, newErr( + codes.Unauthenticated, + "Error authenticating event grid", + err, + ) + } return &EventGridEventService{ client: client, topicClient: topicClient, topicLocation: topicLocation, + accessToken: accessToken, }, nil } From d53fc2787fdb51e5157e689c681b8007d4d3115a Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Tue, 24 Aug 2021 15:59:30 +1000 Subject: [PATCH 45/83] feat: Add client auth --- pkg/plugins/events/eventgrid/eventgrid.go | 86 +++++------------------ 1 file changed, 19 insertions(+), 67 deletions(-) diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go index cd9217962..fe20474c0 100644 --- a/pkg/plugins/events/eventgrid/eventgrid.go +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -18,8 +18,6 @@ import ( "context" "encoding/json" "fmt" - "net/http" - "net/url" "strings" "time" @@ -27,6 +25,8 @@ import ( "github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi" eventgridmgmt "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid" eventgridmgmtapi "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi" + "github.com/Azure/go-autorest/autorest/azure" + eventgridauth "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/Azure/go-autorest/autorest/date" "github.com/nitric-dev/membrane/pkg/plugins/errors" "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" @@ -39,37 +39,6 @@ type EventGridEventService struct { client eventgridapi.BaseClientAPI topicClient eventgridmgmtapi.TopicsClientAPI topicLocation string - accessToken AzureAccessToken -} - -type AzureAccessToken struct { - TokenType string `json:"token_type"` - ExpiresIn string `json:"expires_in"` - ExtExpiresIn string `json:"ext_expires_in"` - ExpiresOn string `json:"expires_on"` - NotBefore string `json:"not_before"` - Resource string `json:"resource"` - AccessToken string `json:"access_token"` -} - -func GetToken(tenantId string, clientId string, clientSecret string) (AzureAccessToken, error) { - requestAccessTokenUri := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/token", tenantId) - requestBody := url.Values{ - "grant_type": {"client_credentials"}, - "client_id": {clientId}, - "client_secret": {clientSecret}, - "resource": {"https://management.azure.com/"}, - } - resp, err := http.PostForm(requestAccessTokenUri, requestBody) - if err != nil { - return AzureAccessToken{}, err - } - - var result AzureAccessToken - - json.NewDecoder(resp.Body).Decode(&result) - - return result, nil } func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.NitricEvent) ([]eventgrid.Event, error) { @@ -95,13 +64,10 @@ func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.N func (s *EventGridEventService) ListTopics() ([]string, error) { newErr := errors.ErrorsWithScope("EventGridEventService.ListTopics") - ctx := context.Background() - ctx.Value(map[string]string{ - "Authorization": fmt.Sprintf("%s %s", s.accessToken.TokenType, s.accessToken.AccessToken), - }) pageLength := int32(20) + ctx := context.Background() results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) if err != nil { @@ -128,9 +94,6 @@ func (s *EventGridEventService) ListTopics() ([]string, error) { func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope("EventGridEventService.Publish") ctx := context.Background() - ctx.Value(map[string]string{ - "Authorization": fmt.Sprintf("%s %s", s.accessToken.TokenType, s.accessToken.AccessToken), - }) if len(topic) == 0 { return newErr( @@ -179,31 +142,7 @@ func New() (events.EventService, error) { newErr := errors.ErrorsWithScope("EventGridEventService.New") topicLocation := utils.GetEnv("AZURE_TOPIC_LOCATION", "") subscriptionID := utils.GetEnv("AZURE_SUBSCRIPTION_ID", "") - tenantId := utils.GetEnv("AZURE_TENANT_ID", "") - clientId := utils.GetEnv("AZURE_CLIENT_ID", "") - clientSecret := utils.GetEnv("AZURE_CLIENT_SECRET", "") - if len(tenantId) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_TENANT_ID not configured", - fmt.Errorf(""), - ) - } - if len(clientId) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_CLIENT_ID not configured", - fmt.Errorf(""), - ) - } - if len(clientSecret) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_CLIENT_SECRET not configured", - fmt.Errorf(""), - ) - } if len(topicLocation) == 0 { return nil, newErr( codes.InvalidArgument, @@ -218,12 +157,26 @@ func New() (events.EventService, error) { fmt.Errorf(""), ) } + env := azure.PublicCloud + //Auth requires: + //AZURE_TENANT_ID: Your Azure tenant ID + //AZURE_CLIENT_ID: Your Azure client ID. This will be an app ID from your AAD. + authorizer, err := eventgridauth.NewAuthorizerFromEnvironmentWithResource(env.ResourceIdentifiers.KeyVault) + if err != nil { + return nil, newErr( + codes.Internal, + "Error authenticating event grid", + err, + ) + } client := eventgrid.New() + client.Authorizer = authorizer + topicClient := eventgridmgmt.NewTopicsClient(subscriptionID) - accessToken, err := GetToken(tenantId, clientId, clientSecret) + topicClient.Authorizer = authorizer if err != nil { return nil, newErr( - codes.Unauthenticated, + codes.Internal, "Error authenticating event grid", err, ) @@ -232,7 +185,6 @@ func New() (events.EventService, error) { client: client, topicClient: topicClient, topicLocation: topicLocation, - accessToken: accessToken, }, nil } From 7eec1c767b54cb2818b9931776570e58413d24f9 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 16 Sep 2021 10:21:48 +1000 Subject: [PATCH 46/83] feat: Update client and management auth --- pkg/membrane/membrane.go | 4 +- pkg/plugins/events/eventgrid/eventgrid.go | 177 +++++++++++----------- pkg/providers/azure/membrane.go | 7 +- pkg/providers/azure/plugin.go | 3 +- 4 files changed, 101 insertions(+), 90 deletions(-) diff --git a/pkg/membrane/membrane.go b/pkg/membrane/membrane.go index 15d03fd3f..21cf3a8e7 100644 --- a/pkg/membrane/membrane.go +++ b/pkg/membrane/membrane.go @@ -119,7 +119,7 @@ func (s *Membrane) createDocumentServer() v1.DocumentServiceServer { } // Create a new Nitric events Server -func (s *Membrane) createeventsServer() v1.EventServiceServer { +func (s *Membrane) createEventsServer() v1.EventServiceServer { return grpc2.NewEventServiceServer(s.eventsPlugin) } @@ -166,7 +166,7 @@ func (s *Membrane) Start() error { documentServer := s.createDocumentServer() v1.RegisterDocumentServiceServer(s.grpcServer, documentServer) - eventsServer := s.createeventsServer() + eventsServer := s.createEventsServer() v1.RegisterEventServiceServer(s.grpcServer, eventsServer) topicServer := s.createTopicServer() diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go index fe20474c0..cf45fdd8a 100644 --- a/pkg/plugins/events/eventgrid/eventgrid.go +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -25,47 +25,31 @@ import ( "github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi" eventgridmgmt "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid" eventgridmgmtapi "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi" + "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" - eventgridauth "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/Azure/go-autorest/autorest/date" "github.com/nitric-dev/membrane/pkg/plugins/errors" "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" "github.com/nitric-dev/membrane/pkg/plugins/events" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" "github.com/nitric-dev/membrane/pkg/utils" ) type EventGridEventService struct { events.UnimplementedeventsPlugin - client eventgridapi.BaseClientAPI - topicClient eventgridmgmtapi.TopicsClientAPI - topicLocation string -} - -func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.NitricEvent) ([]eventgrid.Event, error) { - payload, err := json.Marshal(event.Payload) - if err != nil { - return nil, err - } - subject := fmt.Sprintf("Subject/%s", topic) - eventType := fmt.Sprintf("Type/%s", topic) - azureEvent := []eventgrid.Event{ - { - ID: &event.ID, - Data: &payload, - Topic: &topic, - EventType: &eventType, - Subject: &subject, - EventTime: &date.Time{time.Now()}, - }, - } - - return azureEvent, nil + client eventgridapi.BaseClientAPI + topicClient eventgridmgmtapi.TopicsClientAPI } func (s *EventGridEventService) ListTopics() ([]string, error) { - newErr := errors.ErrorsWithScope("EventGridEventService.ListTopics") - - pageLength := int32(20) + newErr := errors.ErrorsWithScope( + "EventGrid.ListTopics", + map[string]interface{}{ + "list": "topics", + }, + ) + //Set the topic page length + pageLength := int32(10) ctx := context.Background() results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) @@ -73,13 +57,14 @@ func (s *EventGridEventService) ListTopics() ([]string, error) { if err != nil { return nil, newErr( codes.Internal, - "azure list topics error", + "error listing by subscription", err, ) } var topics []string + //Iterate over the topic pages adding their names to the topics slice for results.NotDone() { topicsList := results.Values() for _, topic := range topicsList { @@ -91,107 +76,129 @@ func (s *EventGridEventService) ListTopics() ([]string, error) { return topics, nil } +func (s *EventGridEventService) GetTopicEndpoint(topicName string) (string, error) { + ctx := context.Background() + pageLength := int32(10) + results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) + if err != nil { + return "", fmt.Errorf(err.Error()) + } + + for results.NotDone() { + topicsList := results.Values() + for _, topic := range topicsList { + if *topic.Name == topicName { + return strings.TrimSuffix(strings.TrimPrefix(*topic.Endpoint, "https://"), "/api/events"), nil + } + } + results.Next() + } + return "", fmt.Errorf("topic with provided name could not be found") +} + +func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.NitricEvent) ([]eventgrid.Event, error) { + payload, err := json.Marshal(event.Payload) + if err != nil { + return nil, err + } + dataVersion := "1.0" + azureEvent := []eventgrid.Event{ + { + ID: &event.ID, + Data: &payload, + EventType: &event.PayloadType, + Subject: &topic, + EventTime: &date.Time{time.Now()}, + DataVersion: &dataVersion, + }, + } + + return azureEvent, nil +} + func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) error { - newErr := errors.ErrorsWithScope("EventGridEventService.Publish") + newErr := errors.ErrorsWithScope( + "EventGrid.Publish", + map[string]interface{}{ + "topic": topic, + }, + ) ctx := context.Background() if len(topic) == 0 { return newErr( codes.InvalidArgument, - "provide non-blank topic", - fmt.Errorf("provided invalid topic"), + "provided invalid topic", + fmt.Errorf(""), ) } if event == nil { return newErr( codes.InvalidArgument, - "provide non-nil event", - fmt.Errorf("provided invalid event"), + "provided invalid event", + fmt.Errorf(""), ) } - //Convert topic -> topic1.westus2-1.eventgrid.azure.net - topicHostName := fmt.Sprintf("%s.%s.eventgrid.azure.net", topic, strings.ToLower(s.topicLocation)) - - events, err := s.NitricEventToEvent(topic, event) + topicHostName, err := s.GetTopicEndpoint(topic) if err != nil { return err } - - result, err := s.client.PublishEvents(ctx, topicHostName, events) - + eventToPublish, err := s.NitricEventToEvent(topicHostName, event) if err != nil { return newErr( codes.Internal, - "azure publish event error", + "error marshalling event", err, ) } - if result.StatusCode != 200 { + result, err := s.client.PublishEvents(ctx, topicHostName, eventToPublish) + if err != nil { return newErr( codes.Internal, - "azure publish event returned non-200 status code", - fmt.Errorf(string(rune(result.StatusCode))), + "error publishing event", + err, ) } + + if result.StatusCode != 200 { + return err + } return nil } func New() (events.EventService, error) { - newErr := errors.ErrorsWithScope("EventGridEventService.New") - topicLocation := utils.GetEnv("AZURE_TOPIC_LOCATION", "") subscriptionID := utils.GetEnv("AZURE_SUBSCRIPTION_ID", "") - - if len(topicLocation) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_TOPIC_LOCATION not configured", - fmt.Errorf(""), - ) - } if len(subscriptionID) == 0 { - return nil, newErr( - codes.InvalidArgument, - "AZURE_SUBSCRIPTION_ID not configured", - fmt.Errorf(""), - ) + return nil, fmt.Errorf("AZURE_SUBSCRIPTION_ID not configured") } - env := azure.PublicCloud - //Auth requires: - //AZURE_TENANT_ID: Your Azure tenant ID - //AZURE_CLIENT_ID: Your Azure client ID. This will be an app ID from your AAD. - authorizer, err := eventgridauth.NewAuthorizerFromEnvironmentWithResource(env.ResourceIdentifiers.KeyVault) + + //Get the event grid token, using the event grid resource endpoint + spt, err := azureutils.GetServicePrincipalToken("https://eventgrid.azure.net") if err != nil { - return nil, newErr( - codes.Internal, - "Error authenticating event grid", - err, - ) + return nil, fmt.Errorf("error authenticating event grid client: %v", err.Error()) + } + //Get the event grid management token using the resource management endpoint + mgmtspt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceManagerEndpoint) + if err != nil { + return nil, fmt.Errorf("error authenticating event grid management client: %v", err.Error()) } client := eventgrid.New() - client.Authorizer = authorizer + client.Authorizer = autorest.NewBearerAuthorizer(spt) topicClient := eventgridmgmt.NewTopicsClient(subscriptionID) - topicClient.Authorizer = authorizer - if err != nil { - return nil, newErr( - codes.Internal, - "Error authenticating event grid", - err, - ) - } + topicClient.Authorizer = autorest.NewBearerAuthorizer(mgmtspt) + return &EventGridEventService{ - client: client, - topicClient: topicClient, - topicLocation: topicLocation, + client: client, + topicClient: topicClient, }, nil } func NewWithClient(client eventgridapi.BaseClientAPI, topicClient eventgridmgmtapi.TopicsClientAPI) (events.EventService, error) { return &EventGridEventService{ - client: client, - topicClient: topicClient, - topicLocation: "local1-test", + client: client, + topicClient: topicClient, }, nil } diff --git a/pkg/providers/azure/membrane.go b/pkg/providers/azure/membrane.go index 1503ef036..7b438cbbf 100644 --- a/pkg/providers/azure/membrane.go +++ b/pkg/providers/azure/membrane.go @@ -23,7 +23,7 @@ import ( "github.com/nitric-dev/membrane/pkg/membrane" mongodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/mongodb" - "github.com/nitric-dev/membrane/pkg/plugins/events" + event_grid "github.com/nitric-dev/membrane/pkg/plugins/events/eventgrid" http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" "github.com/nitric-dev/membrane/pkg/plugins/queue" key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" @@ -40,7 +40,10 @@ func main() { fmt.Println("Failed to load document plugin:", err.Error()) } - eventsPlugin := &events.UnimplementedeventsPlugin{} + eventsPlugin, err := event_grid.New() + if err != nil { + fmt.Println("Failed to load event plugin:", err.Error()) + } gatewayPlugin, _ := http_service.New() queuePlugin := &queue.UnimplementedQueuePlugin{} storagePlugin := &storage.UnimplementedStoragePlugin{} diff --git a/pkg/providers/azure/plugin.go b/pkg/providers/azure/plugin.go index 85b452cfa..9e6d6efaa 100644 --- a/pkg/providers/azure/plugin.go +++ b/pkg/providers/azure/plugin.go @@ -18,6 +18,7 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/document" mongodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/mongodb" "github.com/nitric-dev/membrane/pkg/plugins/events" + event_grid "github.com/nitric-dev/membrane/pkg/plugins/events/eventgrid" "github.com/nitric-dev/membrane/pkg/plugins/gateway" http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" "github.com/nitric-dev/membrane/pkg/plugins/queue" @@ -45,7 +46,7 @@ func (p *AzureServiceFactory) NewDocumentService() (document.DocumentService, er // NewEventService - Returns Azure _ based events plugin func (p *AzureServiceFactory) NewEventService() (events.EventService, error) { - return &events.UnimplementedeventsPlugin{}, nil + return event_grid.New() } // NewGatewayService - Returns Azure _ Gateway plugin From d44f959eb86b5a9317b280dfc1d77a582b163d12 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 16 Sep 2021 12:04:14 +1000 Subject: [PATCH 47/83] test: Fix unit tests --- pkg/plugins/events/eventgrid/eventgrid.go | 9 ++- .../events/eventgrid/eventgrid_test.go | 58 +++++++++++-------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go index cf45fdd8a..5fecb069d 100644 --- a/pkg/plugins/events/eventgrid/eventgrid.go +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -162,8 +162,13 @@ func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) ) } - if result.StatusCode != 200 { - return err + statusCode := fmt.Sprint(result.StatusCode) + if strings.Split(statusCode, "")[0] != "2" { + return newErr( + codes.Internal, + "returned non 200 status code", + fmt.Errorf(""), + ) } return nil } diff --git a/pkg/plugins/events/eventgrid/eventgrid_test.go b/pkg/plugins/events/eventgrid/eventgrid_test.go index e667d0d4d..2ddaef40e 100644 --- a/pkg/plugins/events/eventgrid/eventgrid_test.go +++ b/pkg/plugins/events/eventgrid/eventgrid_test.go @@ -16,7 +16,6 @@ package eventgrid_service_test import ( "context" - "fmt" "net/http" eventgridmgmt "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid" @@ -30,6 +29,23 @@ import ( ) var _ = Describe("Event Grid Plugin", func() { + topicName := "Test" + topicEndpoint := "https://Test.local1-test.eventgrid.azure.net/api/events" + topicListResponsePage := eventgridmgmt.NewTopicsListResultPage( + eventgridmgmt.TopicsListResult{ + Value: &[]eventgridmgmt.Topic{ + { + Name: &topicName, + TopicProperties: &eventgridmgmt.TopicProperties{ + Endpoint: &topicEndpoint, + }, + }, + }, + }, + func(context.Context, eventgridmgmt.TopicsListResult) (eventgridmgmt.TopicsListResult, error) { + return eventgridmgmt.TopicsListResult{}, nil + }, + ) When("Listing Available Topics", func() { When("There are no topics available", func() { ctrl := gomock.NewController(GinkgoT()) @@ -56,20 +72,6 @@ var _ = Describe("Event Grid Plugin", func() { topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) - topicName := "Test" - topicListResponsePage := eventgridmgmt.NewTopicsListResultPage( - eventgridmgmt.TopicsListResult{ - Value: &[]eventgridmgmt.Topic{ - { - Name: &topicName, - }, - }, - }, - func(context.Context, eventgridmgmt.TopicsListResult) (eventgridmgmt.TopicsListResult, error) { - return eventgridmgmt.TopicsListResult{}, nil - }, - ) - topicClient.EXPECT().ListBySubscription( gomock.Any(), "", @@ -99,11 +101,11 @@ var _ = Describe("Event Grid Plugin", func() { topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) - eventgridClient.EXPECT().PublishEvents( + topicClient.EXPECT().ListBySubscription( gomock.Any(), - "Test.local1-test.eventgrid.azure.net", + "", gomock.Any(), - ).Return(autorest.Response{}, fmt.Errorf("Topic does not exist")).Times(1) + ).Return(eventgridmgmt.TopicsListResultPage{}, nil).Times(1) It("should return an error", func() { err := eventgridPlugin.Publish("Test", event) @@ -111,7 +113,7 @@ var _ = Describe("Event Grid Plugin", func() { }) }) - When("To a topic that is unauthorised", func() { + When("publishing to a topic that is unauthorised", func() { ctrl := gomock.NewController(GinkgoT()) eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) @@ -123,9 +125,14 @@ var _ = Describe("Event Grid Plugin", func() { gomock.Any(), ).Return(autorest.Response{ &http.Response{ - StatusCode: 404, + StatusCode: 403, }, }, nil).Times(1) + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(topicListResponsePage, nil).Times(1) It("should return an error", func() { err := eventgridPlugin.Publish("Test", event) @@ -145,9 +152,14 @@ var _ = Describe("Event Grid Plugin", func() { gomock.Any(), ).Return(autorest.Response{ &http.Response{ - StatusCode: 200, + StatusCode: 202, }, }, nil).Times(1) + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(topicListResponsePage, nil).Times(1) It("should successfully publish the message", func() { err := eventgridPlugin.Publish("Test", event) @@ -164,7 +176,7 @@ var _ = Describe("Event Grid Plugin", func() { It("should return an error", func() { err := eventgridPlugin.Publish("", event) Expect(err).Should(HaveOccurred()) - Expect(err.Error()).Should(ContainSubstring("provide non-blank topic")) + Expect(err.Error()).Should(ContainSubstring("provided invalid topic")) }) }) @@ -177,7 +189,7 @@ var _ = Describe("Event Grid Plugin", func() { It("should return an error", func() { err := eventgridPlugin.Publish("Test", nil) Expect(err).Should(HaveOccurred()) - Expect(err.Error()).Should(ContainSubstring("provide non-nil event")) + Expect(err.Error()).Should(ContainSubstring("provided invalid event")) }) }) }) From 4ce852240df131e7ffd2c94932688e23b0b2fdbc Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Thu, 16 Sep 2021 12:15:18 +1000 Subject: [PATCH 48/83] feat: update mock generation --- makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/makefile b/makefile index d0c2732cf..fb813530c 100644 --- a/makefile +++ b/makefile @@ -197,6 +197,9 @@ generate-mocks: @mkdir -p mocks/key_vault @mkdir -p mocks/s3 @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/secret_manager/mock.go + @mkdir -p mocks/mock_event_grid @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/key_vault/mock.go - @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go \ No newline at end of file + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go + @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi BaseClientAPI > mocks/mock_event_grid/mock.go + @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi TopicsClientAPI > mocks/mock_event_grid/topic.go \ No newline at end of file From 02489dbc7b1f67ca671604c329924152b414ed4b Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Mon, 20 Sep 2021 09:45:28 +1000 Subject: [PATCH 49/83] feat: rebase onto develop --- go.mod | 3 ++- go.sum | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7f96612cb..6b9d0edb7 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/Azure/go-autorest/autorest v0.11.18 github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/DataDog/zstd v1.4.8 // indirect @@ -38,7 +39,7 @@ require ( golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.5 + golang.org/x/tools v0.1.6 google.golang.org/api v0.40.0 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c google.golang.org/grpc v1.35.0 diff --git a/go.sum b/go.sum index da7ab82df..d6f3a9571 100644 --- a/go.sum +++ b/go.sum @@ -173,7 +173,6 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -277,7 +276,6 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= @@ -343,7 +341,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -404,6 +401,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0 h1:OtISOGfH6sOWa1/qXqqAiOIAO6Z5J3AEAE18WAq6BiQ= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA= @@ -504,6 +503,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -578,6 +578,8 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -650,6 +652,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6 h1:SIasE1FVIQOWz2GEAHFOmoW7xchJcqlucjSULTL0Ag4= +golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From b839e036ac430a1ff221bd13cff9ca28e989906a Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Mon, 20 Sep 2021 16:14:18 +1000 Subject: [PATCH 50/83] feat: Address review feedback --- pkg/plugins/events/eventgrid/eventgrid.go | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go index 5fecb069d..898b870e5 100644 --- a/pkg/plugins/events/eventgrid/eventgrid.go +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -76,7 +76,7 @@ func (s *EventGridEventService) ListTopics() ([]string, error) { return topics, nil } -func (s *EventGridEventService) GetTopicEndpoint(topicName string) (string, error) { +func (s *EventGridEventService) getTopicEndpoint(topicName string) (string, error) { ctx := context.Background() pageLength := int32(10) results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) @@ -96,24 +96,25 @@ func (s *EventGridEventService) GetTopicEndpoint(topicName string) (string, erro return "", fmt.Errorf("topic with provided name could not be found") } -func (s *EventGridEventService) NitricEventToEvent(topic string, event *events.NitricEvent) ([]eventgrid.Event, error) { - payload, err := json.Marshal(event.Payload) - if err != nil { - return nil, err - } - dataVersion := "1.0" - azureEvent := []eventgrid.Event{ - { +func (s *EventGridEventService) nitricEventsToAzureEvents(topic string, events []*events.NitricEvent) ([]eventgrid.Event, error) { + var azureEvents []eventgrid.Event + for _, event := range events { + payload, err := json.Marshal(event.Payload) + if err != nil { + return nil, err + } + dataVersion := "1.0" + azureEvents = append(azureEvents, eventgrid.Event{ ID: &event.ID, Data: &payload, EventType: &event.PayloadType, Subject: &topic, EventTime: &date.Time{time.Now()}, DataVersion: &dataVersion, - }, + }) } - return azureEvent, nil + return azureEvents, nil } func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) error { @@ -129,22 +130,22 @@ func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) return newErr( codes.InvalidArgument, "provided invalid topic", - fmt.Errorf(""), + fmt.Errorf("non-blank topic is required"), ) } if event == nil { return newErr( codes.InvalidArgument, "provided invalid event", - fmt.Errorf(""), + fmt.Errorf("non-nil event is required"), ) } - topicHostName, err := s.GetTopicEndpoint(topic) + topicHostName, err := s.getTopicEndpoint(topic) if err != nil { return err } - eventToPublish, err := s.NitricEventToEvent(topicHostName, event) + eventToPublish, err := s.nitricEventsToAzureEvents(topicHostName, []*events.NitricEvent{event}) if err != nil { return newErr( codes.Internal, @@ -162,8 +163,7 @@ func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) ) } - statusCode := fmt.Sprint(result.StatusCode) - if strings.Split(statusCode, "")[0] != "2" { + if result.StatusCode <= 200 || result.StatusCode >= 300 { return newErr( codes.Internal, "returned non 200 status code", From 7d29e992b79e6a24294270ddc449b6fe208809a3 Mon Sep 17 00:00:00 2001 From: Ryan Cartwright Date: Mon, 20 Sep 2021 16:16:27 +1000 Subject: [PATCH 51/83] feat: fix 200 status code not being included --- pkg/plugins/events/eventgrid/eventgrid.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go index 898b870e5..a8d310545 100644 --- a/pkg/plugins/events/eventgrid/eventgrid.go +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -163,11 +163,11 @@ func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) ) } - if result.StatusCode <= 200 || result.StatusCode >= 300 { + if result.StatusCode < 200 || result.StatusCode >= 300 { return newErr( codes.Internal, "returned non 200 status code", - fmt.Errorf(""), + fmt.Errorf(result.Status), ) } return nil From 13aad99924d25db074b6049be41688a1728f9169 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 21 Sep 2021 12:35:12 +1000 Subject: [PATCH 52/83] ci: Add azure-membrane publishing. --- .github/workflows/publish-binaries.yaml | 9 +++++++++ makefile | 1 + 2 files changed, 10 insertions(+) diff --git a/.github/workflows/publish-binaries.yaml b/.github/workflows/publish-binaries.yaml index eab1e6e3a..190660011 100644 --- a/.github/workflows/publish-binaries.yaml +++ b/.github/workflows/publish-binaries.yaml @@ -51,6 +51,15 @@ jobs: asset_path: ./bin/membrane-gcp asset_name: membrane-gcp asset_content_type: application/octet-stream + - name: Upload Azure + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-azure + asset_name: membrane-azure + asset_content_type: application/octet-stream - name: Upload Digital Ocean uses: actions/upload-release-asset@v1 env: diff --git a/makefile b/makefile index fb813530c..e80ac5dde 100644 --- a/makefile +++ b/makefile @@ -176,6 +176,7 @@ build-all-binaries: clean generate-proto @echo Building all provider membranes @CGO_ENABLED=0 go build -o bin/membrane-gcp -ldflags="-extldflags=-static" ./pkg/providers/gcp/membrane.go @CGO_ENABLED=0 go build -o bin/membrane-aws -ldflags="-extldflags=-static" ./pkg/providers/aws/membrane.go + @CGO_ENABLED=0 go build -o bin/membrane-azure -ldflags="-extldflags=-static" ./pkg/providers/azure/membrane.go @CGO_ENABLED=0 go build -o bin/membrane-do -ldflags="-extldflags=-static" ./pkg/providers/do/membrane.go @CGO_ENABLED=0 go build -o bin/membrane-dev -ldflags="-extldflags=-static" ./pkg/providers/dev/membrane.go From 1ee0711842b50c7efef82372e082490aaa82e24f Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Mon, 20 Sep 2021 09:48:23 +1000 Subject: [PATCH 53/83] refactor: make query functions pure, deduplicate code. --- pkg/plugins/document/dynamodb/dynamodb.go | 160 ++++++++++------------ 1 file changed, 76 insertions(+), 84 deletions(-) diff --git a/pkg/plugins/document/dynamodb/dynamodb.go b/pkg/plugins/document/dynamodb/dynamodb.go index 068728a37..4cdfd55c8 100644 --- a/pkg/plugins/document/dynamodb/dynamodb.go +++ b/pkg/plugins/document/dynamodb/dynamodb.go @@ -261,6 +261,42 @@ func (s *DynamoDocService) Delete(key *document.Key) error { return nil } +func (s *DynamoDocService) query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { + queryResult := &document.QueryResult{ + Documents: make([]document.Document, 0), + } + + var resFunc resultRetriever = s.performQuery + if collection.Parent == nil || collection.Parent.Id == "" { + resFunc = s.performScan + } + + if res, err := resFunc(collection, expressions, limit, pagingToken); err != nil { + return nil, err + } else { + queryResult.Documents = append(queryResult.Documents, res.Documents...) + queryResult.PagingToken = res.PagingToken + } + + remainingLimit := limit - len(queryResult.Documents) + + // If more results available, perform additional queries + for remainingLimit > 0 && + (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { + + if res, err := resFunc(collection, expressions, remainingLimit, queryResult.PagingToken); err != nil { + return nil, err + } else { + queryResult.Documents = append(queryResult.Documents, res.Documents...) + queryResult.PagingToken = res.PagingToken + } + + remainingLimit = limit - len(queryResult.Documents) + } + + return queryResult, nil +} + func (s *DynamoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "DynamoDocService.Query", @@ -285,69 +321,15 @@ func (s *DynamoDocService) Query(collection *document.Collection, expressions [] ) } - queryResult := &document.QueryResult{ - Documents: make([]document.Document, 0), - } - - // If partition key defined then perform a query - if collection.Parent != nil && collection.Parent.Id != "" { - err := s.performQuery(collection, expressions, limit, pagingToken, queryResult) - if err != nil { - return nil, newErr( - codes.Internal, - "query error", - err, - ) - } - - remainingLimit := limit - len(queryResult.Documents) - - // If more results available, perform additional queries - for remainingLimit > 0 && - (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { - - err := s.performQuery(collection, expressions, remainingLimit, queryResult.PagingToken, queryResult) - if err != nil { - return nil, newErr( - codes.Internal, - "query error", - err, - ) - } - - remainingLimit = limit - len(queryResult.Documents) - } - + if res, err := s.query(collection, expressions, limit, pagingToken); err != nil { + return nil, newErr( + codes.Internal, + "query error", + err, + ) } else { - err := s.performScan(collection, expressions, limit, pagingToken, queryResult) - if err != nil { - return nil, newErr( - codes.Internal, - "scan error", - err, - ) - } - - remainingLimit := limit - len(queryResult.Documents) - - // If more results available, perform additional scans - for remainingLimit > 0 && - (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { - - err := s.performScan(collection, expressions, remainingLimit, queryResult.PagingToken, queryResult) - if err != nil { - return nil, newErr( - codes.Internal, - "scan error", - err, - ) - } - - remainingLimit = limit - len(queryResult.Documents) - } + return res, nil } - - return queryResult, nil } // New - Create a new DynamoDB key value plugin implementation @@ -415,16 +397,23 @@ func createItemMap(source map[string]interface{}, key *document.Key) map[string] return newMap } +type resultRetriever = func( + collection *document.Collection, + expressions []document.QueryExpression, + limit int, + pagingToken map[string]string, +) (*document.QueryResult, error) + func (s *DynamoDocService) performQuery( collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string, - queryResult *document.QueryResult) error { +) (*document.QueryResult, error) { if collection.Parent == nil { // Should never occur - return fmt.Errorf("cannot perform query without partion key defined") + return nil, fmt.Errorf("cannot perform query without partion key defined") } // Sort expressions to help map where "A >= %1 AND A <= %2" to DynamoDB expression "A BETWEEN %1 AND %2" @@ -433,7 +422,7 @@ func (s *DynamoDocService) performQuery( tableName, err := s.getTableName(*collection) if err != nil { - return err + return nil, err } input := &dynamodb.QueryInput{ @@ -470,7 +459,7 @@ func (s *DynamoDocService) performQuery( expKey := fmt.Sprintf(":%v%v", exp.Operand, i) valAttrib, err := dynamodbattribute.Marshal(exp.Value) if err != nil { - return fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) + return nil, fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) } input.ExpressionAttributeValues[expKey] = valAttrib } @@ -483,7 +472,7 @@ func (s *DynamoDocService) performQuery( if len(pagingToken) > 0 { startKey, err := dynamodbattribute.MarshalMap(pagingToken) if err != nil { - return fmt.Errorf("error performing query %v: %v", input, err) + return nil, fmt.Errorf("error performing query %v: %v", input, err) } input.SetExclusiveStartKey(startKey) } @@ -493,10 +482,10 @@ func (s *DynamoDocService) performQuery( resp, err := s.client.Query(input) if err != nil { - return fmt.Errorf("error performing query %v: %v", input, err) + return nil, fmt.Errorf("error performing query %v: %v", input, err) } - return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey, queryResult) + return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey) } func (s *DynamoDocService) performScan( @@ -504,7 +493,7 @@ func (s *DynamoDocService) performScan( expressions []document.QueryExpression, limit int, pagingToken map[string]string, - queryResult *document.QueryResult) error { +) (*document.QueryResult, error) { // Sort expressions to help map where "A >= %1 AND A <= %2" to DynamoDB expression "A BETWEEN %1 AND %2" sort.Sort(document.ExpsSort(expressions)) @@ -546,7 +535,7 @@ func (s *DynamoDocService) performScan( expKey := fmt.Sprintf(":%v%v", exp.Operand, i) valAttrib, err := dynamodbattribute.Marshal(exp.Value) if err != nil { - return fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) + return nil, fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) } input.ExpressionAttributeValues[expKey] = valAttrib } @@ -560,7 +549,7 @@ func (s *DynamoDocService) performScan( if len(pagingToken) > 0 { startKey, err := dynamodbattribute.MarshalMap(pagingToken) if err != nil { - return fmt.Errorf("error performing scan %v: %v", input, err) + return nil, fmt.Errorf("error performing scan %v: %v", input, err) } input.SetExclusiveStartKey(startKey) } @@ -569,21 +558,22 @@ func (s *DynamoDocService) performScan( resp, err := s.client.Scan(input) if err != nil { - return fmt.Errorf("error performing scan %v: %v", input, err) + return nil, fmt.Errorf("error performing scan %v: %v", input, err) } - return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey, queryResult) + return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey) } -func marshalQueryResult(collection *document.Collection, items []map[string]*dynamodb.AttributeValue, lastEvaluatedKey map[string]*dynamodb.AttributeValue, queryResult *document.QueryResult) error { - +func marshalQueryResult(collection *document.Collection, items []map[string]*dynamodb.AttributeValue, lastEvaluatedKey map[string]*dynamodb.AttributeValue) (*document.QueryResult, error) { // Unmarshal Dynamo response items + var pTkn map[string]string = nil var valueMaps []map[string]interface{} - err := dynamodbattribute.UnmarshalListOfMaps(items, &valueMaps) - if err != nil { - return fmt.Errorf("error unmarshalling query response: %v", err) + if err := dynamodbattribute.UnmarshalListOfMaps(items, &valueMaps); err != nil { + return nil, fmt.Errorf("error unmarshalling query response: %v", err) } + docs := make([]document.Document, 0, len(valueMaps)) + // Strip keys & append results for _, m := range valueMaps { // Retrieve the original ID on the result @@ -622,20 +612,22 @@ func marshalQueryResult(collection *document.Collection, items []map[string]*dyn }, Content: m, } - queryResult.Documents = append(queryResult.Documents, sdkDoc) + docs = append(docs, sdkDoc) } // Unmarshal lastEvalutedKey var resultPagingToken map[string]string if len(lastEvaluatedKey) > 0 { - err = dynamodbattribute.UnmarshalMap(lastEvaluatedKey, &resultPagingToken) - if err != nil { - return fmt.Errorf("error unmarshalling query lastEvaluatedKey: %v", err) + if err := dynamodbattribute.UnmarshalMap(lastEvaluatedKey, &resultPagingToken); err != nil { + return nil, fmt.Errorf("error unmarshalling query lastEvaluatedKey: %v", err) } - queryResult.PagingToken = resultPagingToken + pTkn = resultPagingToken } - return nil + return &document.QueryResult{ + Documents: docs, + PagingToken: pTkn, + }, nil } func createFilterExpression(expressions []document.QueryExpression) string { From 276f7ecd9f25e67838b4156ea590bf35784bfbf8 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 21 Sep 2021 10:07:22 +1000 Subject: [PATCH 54/83] feat(documents/dynamodb): Add QueryStream to dynamodb plugin. --- pkg/plugins/document/dynamodb/dynamodb.go | 131 ++++++++++++++++++---- 1 file changed, 110 insertions(+), 21 deletions(-) diff --git a/pkg/plugins/document/dynamodb/dynamodb.go b/pkg/plugins/document/dynamodb/dynamodb.go index 4cdfd55c8..9b6d608cb 100644 --- a/pkg/plugins/document/dynamodb/dynamodb.go +++ b/pkg/plugins/document/dynamodb/dynamodb.go @@ -16,6 +16,7 @@ package dynamodb_service import ( "fmt" + "io" "regexp" "sort" "strings" @@ -236,7 +237,7 @@ func (s *DynamoDocService) Delete(key *document.Key) error { if err != nil { return newErr( codes.Internal, - fmt.Sprintf("error performing delete in table"), + "error performing delete in table", err, ) } @@ -247,7 +248,7 @@ func (s *DynamoDocService) Delete(key *document.Key) error { if err != nil { return newErr( codes.Internal, - fmt.Sprintf("error performing delete"), + "error performing delete", err, ) } @@ -278,22 +279,6 @@ func (s *DynamoDocService) query(collection *document.Collection, expressions [] queryResult.PagingToken = res.PagingToken } - remainingLimit := limit - len(queryResult.Documents) - - // If more results available, perform additional queries - for remainingLimit > 0 && - (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { - - if res, err := resFunc(collection, expressions, remainingLimit, queryResult.PagingToken); err != nil { - return nil, err - } else { - queryResult.Documents = append(queryResult.Documents, res.Documents...) - queryResult.PagingToken = res.PagingToken - } - - remainingLimit = limit - len(queryResult.Documents) - } - return queryResult, nil } @@ -321,14 +306,115 @@ func (s *DynamoDocService) Query(collection *document.Collection, expressions [] ) } - if res, err := s.query(collection, expressions, limit, pagingToken); err != nil { + queryResult, err := s.query(collection, expressions, limit, pagingToken) + if err != nil { return nil, newErr( codes.Internal, "query error", err, ) - } else { - return res, nil + } + + remainingLimit := limit - len(queryResult.Documents) + + // If more results available, perform additional queries + for remainingLimit > 0 && + (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { + + if res, err := s.query(collection, expressions, remainingLimit, queryResult.PagingToken); err != nil { + return nil, newErr( + codes.Internal, + "query error", + err, + ) + } else { + queryResult.Documents = append(queryResult.Documents, res.Documents...) + queryResult.PagingToken = res.PagingToken + } + + remainingLimit = limit - len(queryResult.Documents) + } + + return queryResult, nil +} + +func (s *DynamoDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "DynamoDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) + + colErr := document.ValidateQueryCollection(collection) + expErr := document.ValidateExpressions(expressions) + + if colErr != nil || expErr != nil { + // Return an error only iterator + return func() (*document.Document, error) { + return nil, newErr( + codes.InvalidArgument, + "invalid arguments", + fmt.Errorf("collection error:%v, expression error: %v", colErr, expErr), + ) + } + } + + var tmpLimit = limit + var documents []document.Document + var pagingToken map[string]string + + // Initial fetch + res, fetchErr := s.query(collection, expressions, tmpLimit, nil) + + if fetchErr != nil { + // Return an error only iterator if the initial fetch failed + return func() (*document.Document, error) { + return nil, newErr( + codes.Internal, + "query error", + fetchErr, + ) + } + } + + documents = res.Documents + pagingToken = res.PagingToken + + return func() (*document.Document, error) { + // check the iteration state + if tmpLimit <= 0 && limit > 0 { + // we've reached the limit of reading + return nil, io.EOF + } else if pagingToken != nil && len(documents) == 0 { + // we've run out of documents and have more pages to read + res, fetchErr = s.query(collection, expressions, tmpLimit, pagingToken) + documents = res.Documents + pagingToken = res.PagingToken + } else if pagingToken == nil && len(documents) == 0 { + // we're all out of documents and pages before hitting the limit + return nil, io.EOF + } + + // We received an error fetching the docs + if fetchErr != nil { + return nil, newErr( + codes.Internal, + "query error", + fetchErr, + ) + } + + if len(documents) == 0 { + return nil, io.EOF + } + + // pop the first element + var doc document.Document + doc, documents = documents[0], documents[1:] + tmpLimit = tmpLimit - 1 + + return &doc, nil } } @@ -499,6 +585,9 @@ func (s *DynamoDocService) performScan( sort.Sort(document.ExpsSort(expressions)) tableName, err := s.getTableName(*collection) + if err != nil { + return nil, err + } input := &dynamodb.ScanInput{ TableName: tableName, From a4c84cfe3826e51121590a6f4723868be3e2cb84 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Mon, 20 Sep 2021 10:36:37 +1000 Subject: [PATCH 55/83] test(documents/dynamodb): Connect dynamo document plugin to streaming test harness. --- tests/plugins/document/dynamodb/dynamodb_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/plugins/document/dynamodb/dynamodb_test.go b/tests/plugins/document/dynamodb/dynamodb_test.go index 04f5dfd6f..89cd7b158 100644 --- a/tests/plugins/document/dynamodb/dynamodb_test.go +++ b/tests/plugins/document/dynamodb/dynamodb_test.go @@ -87,6 +87,7 @@ var _ = Describe("DynamoDb", func() { test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) func createDynamoClient() *dynamodb.DynamoDB { From 4bba069d92f5b231f381395a629654eb8fee7d5f Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Mon, 6 Sep 2021 12:51:39 +1000 Subject: [PATCH 56/83] feat(azblob): Add azure storage plugin implementation --- go.mod | 4 +- go.sum | 29 +- makefile | 4 +- pkg/plugins/storage/azblob/azblob.go | 180 ++++++++++++ .../storage/azblob/azblob_suite_test.go | 27 ++ pkg/plugins/storage/azblob/azblob_test.go | 262 ++++++++++++++++++ pkg/plugins/storage/azblob/iface/adapters.go | 60 ++++ pkg/plugins/storage/azblob/iface/iface.go | 48 ++++ pkg/providers/azure/membrane.go | 8 +- pkg/providers/azure/plugin.go | 3 +- 10 files changed, 612 insertions(+), 13 deletions(-) create mode 100644 pkg/plugins/storage/azblob/azblob.go create mode 100644 pkg/plugins/storage/azblob/azblob_suite_test.go create mode 100644 pkg/plugins/storage/azblob/azblob_test.go create mode 100644 pkg/plugins/storage/azblob/iface/adapters.go create mode 100644 pkg/plugins/storage/azblob/iface/iface.go diff --git a/go.mod b/go.mod index 6b9d0edb7..719f5f5d3 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/DataDog/zstd v1.4.8 // indirect github.com/Knetic/govaluate v3.0.0+incompatible @@ -25,7 +26,7 @@ require ( github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.3 // indirect github.com/google/addlicense v1.0.0 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.2.0 github.com/googleapis/gax-go/v2 v2.0.5 github.com/mitchellh/mapstructure v1.4.1 github.com/onsi/ginkgo v1.16.4 @@ -45,4 +46,5 @@ require ( google.golang.org/grpc v1.35.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.26.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index d6f3a9571..ccc3b04dc 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,12 @@ cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= @@ -53,10 +57,14 @@ github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyC github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.15 h1:X+p2GF0GWyOiSmqohIaEeuNFNDY4I4EOlVuUQvFdWMk= +github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 h1:DOhB+nXkF7LN0JfBGB5YtCF6QLK8mLe4psaHF7ZQEKM= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.3/go.mod h1:yAQ2b6eP/CmLPnmLvxtT1ALIY3OR1oFcCqVBi8vHiTc= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -122,7 +130,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= github.com/envoyproxy/protoc-gen-validate v0.6.1/go.mod h1:txg5va2Qkip90uYoSKH+nkAAmXrb2j3iq4FLwdrCbXQ= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -172,6 +179,8 @@ github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZC github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -242,8 +251,9 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2 h1:LR89qFljJ48s990kEK github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -286,8 +296,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -300,6 +311,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSW github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= @@ -330,7 +343,6 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7T github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= @@ -480,6 +492,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -502,6 +515,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= @@ -548,6 +562,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -565,6 +580,7 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -757,8 +773,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= diff --git a/makefile b/makefile index e80ac5dde..99e60e031 100644 --- a/makefile +++ b/makefile @@ -197,9 +197,11 @@ generate-mocks: @mkdir -p mocks/secrets_manager @mkdir -p mocks/key_vault @mkdir -p mocks/s3 - @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/secret_manager/mock.go + @mkdir -p mocks/azblob @mkdir -p mocks/mock_event_grid + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/secret_manager/mock.go @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/storage/azblob/iface AzblobServiceUrlIface,AzblobContainerUrlIface,AzblobBlockBlobUrlIface,AzblobDownloadResponse > mocks/azblob/mock.go @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/key_vault/mock.go @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi BaseClientAPI > mocks/mock_event_grid/mock.go diff --git a/pkg/plugins/storage/azblob/azblob.go b/pkg/plugins/storage/azblob/azblob.go new file mode 100644 index 000000000..dae98339f --- /dev/null +++ b/pkg/plugins/storage/azblob/azblob.go @@ -0,0 +1,180 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/url" + "time" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" + "github.com/nitric-dev/membrane/pkg/plugins/storage" + azblob_service_iface "github.com/nitric-dev/membrane/pkg/plugins/storage/azblob/iface" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" + "github.com/nitric-dev/membrane/pkg/utils" +) + +// AzblobStorageService - Nitric membrane storage plugin implementation for Azure Storage +type AzblobStorageService struct { + client azblob_service_iface.AzblobServiceUrlIface +} + +func (a *AzblobStorageService) getBlobUrl(bucket string, key string) azblob_service_iface.AzblobBlockBlobUrlIface { + cUrl := a.client.NewContainerURL(bucket) + // Get a new blob for the key name + return cUrl.NewBlockBlobURL(key) +} + +func (a *AzblobStorageService) Read(bucket string, key string) ([]byte, error) { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.Read", + fmt.Sprintf("bucket=%s", bucket), + fmt.Sprintf("key=%s", key), + ) + // Get the bucket for this bucket name + blob := a.getBlobUrl(bucket, key) + //// download the blob + r, err := blob.Download( + context.TODO(), + 0, + azblob.CountToEnd, + azblob.BlobAccessConditions{}, + false, + azblob.ClientProvidedKeyOptions{}, + ) + + if err != nil { + return nil, newErr( + codes.Internal, + "Unable to download blob", + err, + ) + } + + // TODO: Configure retries + data := r.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20}) + + return ioutil.ReadAll(data) +} + +func (a *AzblobStorageService) Write(bucket string, key string, object []byte) error { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.Write", + fmt.Sprintf("bucket=%s", bucket), + fmt.Sprintf("key=%s", key), + ) + + blob := a.getBlobUrl(bucket, key) + + if _, err := blob.Upload( + context.TODO(), + bytes.NewReader(object), + azblob.BlobHTTPHeaders{}, + azblob.Metadata{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil, + azblob.ClientProvidedKeyOptions{}, + ); err != nil { + return newErr( + codes.Internal, + "Unable to write blob data", + err, + ) + } + + return nil +} + +func (a *AzblobStorageService) Delete(bucket string, key string) error { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.Delete", + fmt.Sprintf("bucket=%s", bucket), + fmt.Sprintf("key=%s", key), + ) + + // Get the bucket for this bucket name + blob := a.getBlobUrl(bucket, key) + + if _, err := blob.Delete( + context.TODO(), + azblob.DeleteSnapshotsOptionInclude, + azblob.BlobAccessConditions{}, + ); err != nil { + return newErr( + codes.Internal, + "Unable to delete blob", + err, + ) + } + + return nil +} + +const expiryBuffer = 2 * time.Minute + +func tokenRefresherFromSpt(spt *adal.ServicePrincipalToken) azblob.TokenRefresher { + return func(credential azblob.TokenCredential) time.Duration { + if err := spt.Refresh(); err != nil { + fmt.Println("Error refreshing token: ", err) + } else { + tkn := spt.Token() + credential.SetToken(tkn.AccessToken) + + return tkn.Expires().Sub(time.Now().Add(expiryBuffer)) + } + + // Mark the token as already expired + return time.Duration(0) + } +} + +// New - Creates a new instance of the AzblobStorageService +func New() (storage.StorageService, error) { + // TODO: Create a default storage account for the stack??? + // XXX: This will limit a membrane wrapped application + // to accessing a single storage account + storageAccount := utils.GetEnv("AZURE_STORAGE_ACCOUNT", "") + + if storageAccount == "" { + return nil, fmt.Errorf("AZURE_STORAGE_ACCOUNT not configured") + } + + spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.Storage) + if err != nil { + return nil, err + } + + cTkn := azblob.NewTokenCredential(spt.Token().AccessToken, tokenRefresherFromSpt(spt)) + + var accountURL *url.URL + if accountURL, err = url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", storageAccount)); err != nil { + return nil, err + } + + pipeline := azblob.NewPipeline(cTkn, azblob.PipelineOptions{}) + client := azblob.NewServiceURL(*accountURL, pipeline) + + return &AzblobStorageService{ + client: azblob_service_iface.AdaptServiceUrl(client), + }, nil +} diff --git a/pkg/plugins/storage/azblob/azblob_suite_test.go b/pkg/plugins/storage/azblob/azblob_suite_test.go new file mode 100644 index 000000000..a1f080c59 --- /dev/null +++ b/pkg/plugins/storage/azblob/azblob_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestAzblob(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Azblob Suite") +} diff --git a/pkg/plugins/storage/azblob/azblob_test.go b/pkg/plugins/storage/azblob/azblob_test.go new file mode 100644 index 000000000..58fc99b1e --- /dev/null +++ b/pkg/plugins/storage/azblob/azblob_test.go @@ -0,0 +1,262 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service + +import ( + "bytes" + "fmt" + "io/ioutil" + "strings" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + mock_azblob "github.com/nitric-dev/membrane/mocks/azblob" +) + +var _ = Describe("Azblob", func() { + //Context("New", func() { + // When("", func() { + + // }) + //}) + + Context("Read", func() { + When("Azure returns a successfuly response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + mockDown := mock_azblob.NewMockAzblobDownloadResponse(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully return the read payload", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Download once on the blob with the expected options") + mockBlob.EXPECT().Download( + gomock.Any(), + int64(0), + int64(0), + azblob.BlobAccessConditions{}, + false, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(mockDown, nil) + + By("Reading from the download response") + mockDown.EXPECT().Body(gomock.Any()).Times(1).Return(ioutil.NopCloser(strings.NewReader("file-contents"))) + + data, err := storagePlugin.Read("my-bucket", "my-blob") + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Returning the read data") + Expect(data).To(BeEquivalentTo([]byte("file-contents"))) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should return an error", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Download once on the blob with the expected options") + mockBlob.EXPECT().Download( + gomock.Any(), + int64(0), + int64(0), + azblob.BlobAccessConditions{}, + false, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(nil, fmt.Errorf("Failed to download")) + + _, err := storagePlugin.Read("my-bucket", "my-blob") + + By("Returning an error") + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Context("Write", func() { + When("Azure returns a successful response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully write the blob", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Upload( + gomock.Any(), + bytes.NewReader([]byte("test")), + azblob.BlobHTTPHeaders{}, + azblob.Metadata{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(&azblob.BlockBlobUploadResponse{}, nil) + + err := storagePlugin.Write("my-bucket", "my-blob", []byte("test")) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should return an error", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Upload( + gomock.Any(), + bytes.NewReader([]byte("test")), + azblob.BlobHTTPHeaders{}, + azblob.Metadata{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(nil, fmt.Errorf("mock-error")) + + err := storagePlugin.Write("my-bucket", "my-blob", []byte("test")) + + By("returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) + + Context("Delete", func() { + When("Azure returns a successful response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully write the blob", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Delete( + gomock.Any(), + azblob.DeleteSnapshotsOptionInclude, + azblob.BlobAccessConditions{}, + ).Times(1).Return(&azblob.BlobDeleteResponse{}, nil) + + err := storagePlugin.Delete("my-bucket", "my-blob") + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully write the blob", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Delete( + gomock.Any(), + azblob.DeleteSnapshotsOptionInclude, + azblob.BlobAccessConditions{}, + ).Times(1).Return(nil, fmt.Errorf("mock-error")) + + err := storagePlugin.Delete("my-bucket", "my-blob") + + By("Not returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) +}) diff --git a/pkg/plugins/storage/azblob/iface/adapters.go b/pkg/plugins/storage/azblob/iface/adapters.go new file mode 100644 index 000000000..0318cbdb6 --- /dev/null +++ b/pkg/plugins/storage/azblob/iface/adapters.go @@ -0,0 +1,60 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service_iface + +import ( + "context" + "io" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +func AdaptServiceUrl(c azblob.ServiceURL) AzblobServiceUrlIface { + return serviceUrl{c} +} + +func AdaptContainerUrl(c azblob.ContainerURL) AzblobContainerUrlIface { + return containerUrl{c} +} + +func AdaptBlobUrl(c azblob.BlockBlobURL) AzblobBlockBlobUrlIface { + return blobUrl{c} +} + +type ( + serviceUrl struct{ c azblob.ServiceURL } + containerUrl struct{ c azblob.ContainerURL } + blobUrl struct{ c azblob.BlockBlobURL } +) + +func (c serviceUrl) NewContainerURL(bucket string) AzblobContainerUrlIface { + return AdaptContainerUrl(c.c.NewContainerURL(bucket)) +} + +func (c containerUrl) NewBlockBlobURL(blob string) AzblobBlockBlobUrlIface { + return AdaptBlobUrl(c.c.NewBlockBlobURL(blob)) +} + +func (c blobUrl) Download(ctx context.Context, offset int64, count int64, bac azblob.BlobAccessConditions, f bool, cpk azblob.ClientProvidedKeyOptions) (AzblobDownloadResponse, error) { + return c.c.Download(ctx, offset, count, bac, f, cpk) +} + +func (c blobUrl) Upload(ctx context.Context, r io.ReadSeeker, h azblob.BlobHTTPHeaders, m azblob.Metadata, bac azblob.BlobAccessConditions, att azblob.AccessTierType, btm azblob.BlobTagsMap, cpk azblob.ClientProvidedKeyOptions) (*azblob.BlockBlobUploadResponse, error) { + return c.c.Upload(ctx, r, h, m, bac, att, btm, cpk) +} + +func (c blobUrl) Delete(ctx context.Context, dot azblob.DeleteSnapshotsOptionType, bac azblob.BlobAccessConditions) (*azblob.BlobDeleteResponse, error) { + return c.c.Delete(ctx, dot, bac) +} diff --git a/pkg/plugins/storage/azblob/iface/iface.go b/pkg/plugins/storage/azblob/iface/iface.go new file mode 100644 index 000000000..37f0fa65d --- /dev/null +++ b/pkg/plugins/storage/azblob/iface/iface.go @@ -0,0 +1,48 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service_iface + +import ( + "context" + "io" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +// AzblobServiceUrlIface - Mockable client interface +// for azblob.ServiceUrl +type AzblobServiceUrlIface interface { + NewContainerURL(string) AzblobContainerUrlIface +} + +// AzblobContainerUrlIface - Mockable client interface +// for azblob.ContainerUrl +type AzblobContainerUrlIface interface { + NewBlockBlobURL(string) AzblobBlockBlobUrlIface +} + +// AzblobBlockBlobUrlIface - Mockable client interface +// for azblob.BlockBlobUrl +type AzblobBlockBlobUrlIface interface { + Download(context.Context, int64, int64, azblob.BlobAccessConditions, bool, azblob.ClientProvidedKeyOptions) (AzblobDownloadResponse, error) + Upload(context.Context, io.ReadSeeker, azblob.BlobHTTPHeaders, azblob.Metadata, azblob.BlobAccessConditions, azblob.AccessTierType, azblob.BlobTagsMap, azblob.ClientProvidedKeyOptions) (*azblob.BlockBlobUploadResponse, error) + Delete(context.Context, azblob.DeleteSnapshotsOptionType, azblob.BlobAccessConditions) (*azblob.BlobDeleteResponse, error) +} + +// AzblobDownloadResponse - Mockable client interface +// for azblob.DownloadResponse +type AzblobDownloadResponse interface { + Body(azblob.RetryReaderOptions) io.ReadCloser +} diff --git a/pkg/providers/azure/membrane.go b/pkg/providers/azure/membrane.go index 7b438cbbf..fffe31336 100644 --- a/pkg/providers/azure/membrane.go +++ b/pkg/providers/azure/membrane.go @@ -27,7 +27,7 @@ import ( http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" "github.com/nitric-dev/membrane/pkg/plugins/queue" key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" - "github.com/nitric-dev/membrane/pkg/plugins/storage" + azblob_service "github.com/nitric-dev/membrane/pkg/plugins/storage/azblob" ) func main() { @@ -46,7 +46,7 @@ func main() { } gatewayPlugin, _ := http_service.New() queuePlugin := &queue.UnimplementedQueuePlugin{} - storagePlugin := &storage.UnimplementedStoragePlugin{} + storagePlugin, _ := azblob_service.New() secretPlugin, err := key_vault.New() if err != nil { fmt.Println("Failed to load secret plugin:", err.Error()) @@ -73,9 +73,9 @@ func main() { select { case membraneError := <-errChan: - fmt.Println(fmt.Sprintf("Membrane Error: %v, exiting", membraneError)) + fmt.Printf("Membrane Error: %v, exiting\n", membraneError) case sigTerm := <-term: - fmt.Println(fmt.Sprintf("Received %v, exiting", sigTerm)) + fmt.Printf("Received %v, exiting\n", sigTerm) } m.Stop() diff --git a/pkg/providers/azure/plugin.go b/pkg/providers/azure/plugin.go index 9e6d6efaa..74c496880 100644 --- a/pkg/providers/azure/plugin.go +++ b/pkg/providers/azure/plugin.go @@ -25,6 +25,7 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/secret" key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" "github.com/nitric-dev/membrane/pkg/plugins/storage" + azblob_service "github.com/nitric-dev/membrane/pkg/plugins/storage/azblob" "github.com/nitric-dev/membrane/pkg/providers" ) @@ -61,5 +62,5 @@ func (p *AzureServiceFactory) NewQueueService() (queue.QueueService, error) { // NewStorageService - Returns Azure _ based storage plugin func (p *AzureServiceFactory) NewStorageService() (storage.StorageService, error) { - return &storage.UnimplementedStoragePlugin{}, nil + return azblob_service.New() } From 49b7d1722de555eade006ee0c7fe7dc2ce888d01 Mon Sep 17 00:00:00 2001 From: Jye Cusch Date: Mon, 13 Sep 2021 09:57:14 +1000 Subject: [PATCH 57/83] feat(azqueue): azure storage queue plugin implementation --- go.mod | 3 +- go.sum | 10 +- pkg/plugins/queue/azqueue/azqueue.go | 281 ++++++++++++++++++++ pkg/plugins/queue/azqueue/iface/adapters.go | 69 +++++ pkg/plugins/queue/azqueue/iface/iface.go | 40 +++ pkg/plugins/storage/azblob/azblob.go | 9 +- pkg/providers/azure/membrane.go | 5 +- pkg/providers/azure/utils/const.go | 18 ++ 8 files changed, 421 insertions(+), 14 deletions(-) create mode 100644 pkg/plugins/queue/azqueue/azqueue.go create mode 100644 pkg/plugins/queue/azqueue/iface/adapters.go create mode 100644 pkg/plugins/queue/azqueue/iface/iface.go create mode 100644 pkg/providers/azure/utils/const.go diff --git a/go.mod b/go.mod index 719f5f5d3..665599f39 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/valyala/fasthttp v1.23.0 github.com/vmihailenco/msgpack v3.3.3+incompatible // indirect go.etcd.io/bbolt v1.3.5 - go.mongodb.org/mongo-driver v1.7.1 // indirect + go.mongodb.org/mongo-driver v1.7.1 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 @@ -46,5 +46,4 @@ require ( google.golang.org/grpc v1.35.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.26.0 - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index ccc3b04dc..0afbccf12 100644 --- a/go.sum +++ b/go.sum @@ -42,12 +42,15 @@ cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= +github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= @@ -181,7 +184,6 @@ github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -331,8 +333,6 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= @@ -343,6 +343,7 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7T github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= @@ -388,8 +389,6 @@ github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/uw-labs/lichen v0.1.4 h1:ZdTT7u0I3MI65CCDaNpMBh3VzhOf1Xbw0gF2VRKQ7/4= -github.com/uw-labs/lichen v0.1.4 h1:ZdTT7u0I3MI65CCDaNpMBh3VzhOf1Xbw0gF2VRKQ7/4= -github.com/uw-labs/lichen v0.1.4/go.mod h1:Fljba7ubiwHbEp1nzUjth+cWUbyQe73WUyL1++5HRNY= github.com/uw-labs/lichen v0.1.4/go.mod h1:Fljba7ubiwHbEp1nzUjth+cWUbyQe73WUyL1++5HRNY= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -773,6 +772,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/plugins/queue/azqueue/azqueue.go b/pkg/plugins/queue/azqueue/azqueue.go new file mode 100644 index 000000000..9054a17a0 --- /dev/null +++ b/pkg/plugins/queue/azqueue/azqueue.go @@ -0,0 +1,281 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure_storage_queue_service + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "time" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + azqueueserviceiface "github.com/nitric-dev/membrane/pkg/plugins/queue/azqueue/iface" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" + + "github.com/nitric-dev/membrane/pkg/utils" + + "github.com/Azure/azure-storage-queue-go/azqueue" + + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" + "github.com/nitric-dev/membrane/pkg/plugins/queue" +) + +// Set to 30 seconds, +const defaultVisibilityTimeout = 30 * time.Second + +type AzureStorageQueueService struct { + client azqueueserviceiface.AzqueueServiceUrlIface +} + +// Returns an adapted azqueue MessagesUrl, which is a client for interacting with messages in a specific queue +func (s *AzureStorageQueueService) getMessagesUrl(queue string) azqueueserviceiface.AzqueueMessageUrlIface { + qUrl := s.client.NewQueueURL(queue) + // Get a new messages URL (used to interact with messages in the queue) + return qUrl.NewMessageURL() +} + +// Returns an adapted azqueue MessageIdUrl, which is a client for interacting with a specific message (task) in a specific queue +func (s *AzureStorageQueueService) getMessageIdUrl(queue string, messageId azqueue.MessageID) azqueueserviceiface.AzqueueMessageIdUrlIface { + mUrl := s.getMessagesUrl(queue) + + return mUrl.NewMessageIDURL(messageId) +} + +func (s *AzureStorageQueueService) Send(queue string, task queue.NitricTask) error { + newErr := errors.ErrorsWithScope( + "AzqueueService.Send", + fmt.Sprintf("queue=%s", queue), + ) + + messages := s.getMessagesUrl(queue) + + // Send the tasks to the queue + if taskBytes, err := json.Marshal(task); err == nil { + ctx := context.TODO() + if _, err := messages.Enqueue(ctx, string(taskBytes), 0, 0); err != nil { + return newErr( + codes.Internal, + "error sending task to queue", + err, + ) + } + } else { + return newErr( + codes.Internal, + "error marshalling the task", + err, + ) + } + + return nil +} + +func (s *AzureStorageQueueService) SendBatch(queueName string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { + failedTasks := make([]*queue.FailedTask, 0) + + for _, task := range tasks { + // Azure Storage Queues don't support batches, so each task must be sent individually. + err := s.Send(queueName, task) + if err != nil { + failedTasks = append(failedTasks, &queue.FailedTask{ + Task: &task, + Message: err.Error(), + }) + } + } + + return &queue.SendBatchResponse{ + FailedTasks: failedTasks, + }, nil +} + +// AzureQueueItemLease - Represents a lease of an Azure Storages Queues item +// Azure requires a combination of their unique reference for a queue item (id) and a pop receipt (lease id) +// to perform operations on the item, such as delete it from the queue. +type AzureQueueItemLease struct { + // The ID of the queue item + // note: this is an id generated by Azure, it's not the user provided unique id. + ID string + // lease id, a new popReceipt is generated each time an item is dequeued. + PopReceipt string +} + +// String - convert the item lease struct to a string, to be returned as a NitricTask LeaseID +func (l *AzureQueueItemLease) String() (string, error) { + leaseID, err := json.Marshal(l) + return string(leaseID), err +} + +// leaseFromString - Unmarshal a NitricTask Lease ID (JSON) to an AzureQueueItemLease +func leaseFromString(leaseID string) (*AzureQueueItemLease, error) { + var lease AzureQueueItemLease + err := json.Unmarshal([]byte(leaseID), &lease) + if err != nil { + return nil, err + } + return &lease, nil +} + +// Receive - Receives a collection of tasks off a given queue. +func (s *AzureStorageQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { + newErr := errors.ErrorsWithScope( + "AzureStorageQueueService.Receive", + fmt.Sprintf("options=%v", options), + ) + + if err := options.Validate(); err != nil { + return nil, newErr( + codes.InvalidArgument, + "invalid receive options provided", + err, + ) + } + + messages := s.getMessagesUrl(options.QueueName) + + ctx := context.TODO() + dequeueResp, err := messages.Dequeue(ctx, int32(*options.Depth), defaultVisibilityTimeout) + if err != nil { + return nil, newErr( + codes.Internal, + "failed to received messages from the queue", + err, + ) + } + + if dequeueResp.NumMessages() == 0 { + return []queue.NitricTask{}, nil + } + + // Convert the Azure Storage Queues messages into Nitric tasks + var tasks []queue.NitricTask + for i := int32(0); i < dequeueResp.NumMessages(); i++ { + m := dequeueResp.Message(i) + var nitricTask queue.NitricTask + err := json.Unmarshal([]byte(m.Text), &nitricTask) + if err != nil { + // TODO: append error to error list and Nack the message. + continue + } + + lease := AzureQueueItemLease{ + ID: m.ID.String(), + PopReceipt: m.PopReceipt.String(), + } + leaseID, err := lease.String() + // This should never happen, it's a fatal error + if err != nil { + return nil, newErr( + codes.Internal, + "failed to construct queue item lease id", + err, + ) + } + + tasks = append(tasks, queue.NitricTask{ + ID: nitricTask.ID, + Payload: nitricTask.Payload, + PayloadType: nitricTask.PayloadType, + LeaseID: leaseID, + }) + } + + return tasks, nil +} + +// Complete - Completes a previously popped queue item +func (s *AzureStorageQueueService) Complete(queue string, leaseId string) error { + newErr := errors.ErrorsWithScope( + "AzureStorageQueueService.Complete", + fmt.Sprintf("queue=%s", queue), + ) + + lease, err := leaseFromString(leaseId) + if err != nil { + return newErr( + codes.InvalidArgument, + "failed to unmarshal lease id value", + err, + ) + } + + // Client for the specific message referenced by the lease + task := s.getMessageIdUrl(queue, azqueue.MessageID(lease.ID)) + ctx := context.TODO() + _, err = task.Delete(ctx, azqueue.PopReceipt(lease.PopReceipt)) + if err != nil { + return newErr( + codes.Internal, + "failed to complete task", + err, + ) + } + + return nil +} + +const expiryBuffer = 2 * time.Minute + +func tokenRefresherFromSpt(spt *adal.ServicePrincipalToken) azqueue.TokenRefresher { + return func(credential azqueue.TokenCredential) time.Duration { + if err := spt.Refresh(); err != nil { + fmt.Println("Error refreshing token: ", err) + } else { + tkn := spt.Token() + credential.SetToken(tkn.AccessToken) + + return tkn.Expires().Sub(time.Now().Add(expiryBuffer)) + } + + // Mark the token as already expired + return time.Duration(0) + } +} + +// New - Constructs a new Azure Storage Queues client with defaults +func New() (queue.QueueService, error) { + storageAccountName := utils.GetEnv(azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV, "") + if storageAccountName == "" { + return nil, fmt.Errorf("failed to determine Azure Storage Account Name, environment variable %s not set", azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV) + } + + spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.Storage) + if err != nil { + return nil, err + } + + cTkn := azqueue.NewTokenCredential(spt.Token().AccessToken, tokenRefresherFromSpt(spt)) + + var accountURL *url.URL + if accountURL, err = url.Parse(fmt.Sprintf("https://%s.queue.core.windows.net", storageAccountName)); err != nil { + return nil, err + } + + pipeline := azqueue.NewPipeline(cTkn, azqueue.PipelineOptions{}) + client := azqueue.NewServiceURL(*accountURL, pipeline) + + return &AzureStorageQueueService{ + client: azqueueserviceiface.AdaptServiceUrl(client), + }, nil +} + +func NewWithClient(client azqueueserviceiface.AzqueueServiceUrlIface) queue.QueueService { + return &AzureStorageQueueService{ + client: client, + } +} diff --git a/pkg/plugins/queue/azqueue/iface/adapters.go b/pkg/plugins/queue/azqueue/iface/adapters.go new file mode 100644 index 000000000..3cae3bf09 --- /dev/null +++ b/pkg/plugins/queue/azqueue/iface/adapters.go @@ -0,0 +1,69 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service_iface + +import ( + "context" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" +) + +func AdaptServiceUrl(c azqueue.ServiceURL) AzqueueServiceUrlIface { + return serviceUrl{c} +} + +func AdaptQueueUrl(c azqueue.QueueURL) AzqueueQueueUrlIface { + return queueUrl{c} +} + +func AdaptMessageUrl(c azqueue.MessagesURL) AzqueueMessageUrlIface { + return messageUrl{c} +} + +func AdaptMessageIdUrl(c azqueue.MessageIDURL) AzqueueMessageIdUrlIface { + return messageIdUrl{c} +} + +type ( + serviceUrl struct{ c azqueue.ServiceURL } + queueUrl struct{ c azqueue.QueueURL } + messageUrl struct{ c azqueue.MessagesURL } + messageIdUrl struct{ c azqueue.MessageIDURL } +) + +func (c serviceUrl) NewQueueURL(queueName string) AzqueueQueueUrlIface { + return AdaptQueueUrl(c.c.NewQueueURL(queueName)) +} + +func (c queueUrl) NewMessageURL() AzqueueMessageUrlIface { + return AdaptMessageUrl(c.c.NewMessagesURL()) +} + +func (c messageUrl) Enqueue(ctx context.Context, messageText string, visibilityTimeout time.Duration, timeToLive time.Duration) (*azqueue.EnqueueMessageResponse, error) { + return c.c.Enqueue(ctx, messageText, visibilityTimeout, timeToLive) +} + +func (c messageUrl) Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (*azqueue.DequeuedMessagesResponse, error) { + return c.c.Dequeue(ctx, maxMessages, visibilityTimeout) +} + +func (c messageUrl) NewMessageIDURL(messageId azqueue.MessageID) AzqueueMessageIdUrlIface { + return AdaptMessageIdUrl(c.c.NewMessageIDURL(messageId)) +} + +func (c messageIdUrl) Delete(ctx context.Context, popReceipt azqueue.PopReceipt) (*azqueue.MessageIDDeleteResponse, error) { + return c.c.Delete(ctx, popReceipt) +} diff --git a/pkg/plugins/queue/azqueue/iface/iface.go b/pkg/plugins/queue/azqueue/iface/iface.go new file mode 100644 index 000000000..2be51ad26 --- /dev/null +++ b/pkg/plugins/queue/azqueue/iface/iface.go @@ -0,0 +1,40 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service_iface + +import ( + "context" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" +) + +type AzqueueServiceUrlIface interface { + NewQueueURL(string) AzqueueQueueUrlIface +} + +type AzqueueQueueUrlIface interface { + NewMessageURL() AzqueueMessageUrlIface +} + +type AzqueueMessageUrlIface interface { + Enqueue(ctx context.Context, messageText string, visibilityTimeout time.Duration, timeToLive time.Duration) (*azqueue.EnqueueMessageResponse, error) + Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (*azqueue.DequeuedMessagesResponse, error) + NewMessageIDURL(messageId azqueue.MessageID) AzqueueMessageIdUrlIface +} + +type AzqueueMessageIdUrlIface interface { + Delete(ctx context.Context, popReceipt azqueue.PopReceipt) (*azqueue.MessageIDDeleteResponse, error) +} diff --git a/pkg/plugins/storage/azblob/azblob.go b/pkg/plugins/storage/azblob/azblob.go index dae98339f..a53e6b4e3 100644 --- a/pkg/plugins/storage/azblob/azblob.go +++ b/pkg/plugins/storage/azblob/azblob.go @@ -153,10 +153,9 @@ func New() (storage.StorageService, error) { // TODO: Create a default storage account for the stack??? // XXX: This will limit a membrane wrapped application // to accessing a single storage account - storageAccount := utils.GetEnv("AZURE_STORAGE_ACCOUNT", "") - - if storageAccount == "" { - return nil, fmt.Errorf("AZURE_STORAGE_ACCOUNT not configured") + storageAccountName := utils.GetEnv(azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV, "") + if storageAccountName == "" { + return nil, fmt.Errorf("failed to determine Azure Storage Account Name, environment variable %s not set", azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV) } spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.Storage) @@ -167,7 +166,7 @@ func New() (storage.StorageService, error) { cTkn := azblob.NewTokenCredential(spt.Token().AccessToken, tokenRefresherFromSpt(spt)) var accountURL *url.URL - if accountURL, err = url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", storageAccount)); err != nil { + if accountURL, err = url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", storageAccountName)); err != nil { return nil, err } diff --git a/pkg/providers/azure/membrane.go b/pkg/providers/azure/membrane.go index fffe31336..6f31fd451 100644 --- a/pkg/providers/azure/membrane.go +++ b/pkg/providers/azure/membrane.go @@ -21,11 +21,12 @@ import ( "os/signal" "syscall" + azqueue_service "github.com/nitric-dev/membrane/pkg/plugins/queue/azqueue" + "github.com/nitric-dev/membrane/pkg/membrane" mongodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/mongodb" event_grid "github.com/nitric-dev/membrane/pkg/plugins/events/eventgrid" http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" - "github.com/nitric-dev/membrane/pkg/plugins/queue" key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" azblob_service "github.com/nitric-dev/membrane/pkg/plugins/storage/azblob" ) @@ -45,7 +46,7 @@ func main() { fmt.Println("Failed to load event plugin:", err.Error()) } gatewayPlugin, _ := http_service.New() - queuePlugin := &queue.UnimplementedQueuePlugin{} + queuePlugin, _ := azqueue_service.New() storagePlugin, _ := azblob_service.New() secretPlugin, err := key_vault.New() if err != nil { diff --git a/pkg/providers/azure/utils/const.go b/pkg/providers/azure/utils/const.go new file mode 100644 index 000000000..9c0d3210b --- /dev/null +++ b/pkg/providers/azure/utils/const.go @@ -0,0 +1,18 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +// AZURE_STORAGE_ACCOUNT_NAME_ENV - Default Azure Storage Account Name for Nitric Stack +const AZURE_STORAGE_ACCOUNT_NAME_ENV = "AZURE_STORAGE_ACCOUNT_NAME" From 640f3e18214570be50604082b3fa321ed299db5e Mon Sep 17 00:00:00 2001 From: Jye Cusch Date: Mon, 13 Sep 2021 16:03:38 +1000 Subject: [PATCH 58/83] test(azqueue): Add tests for azure storage queues plugin --- makefile | 4 +- pkg/plugins/queue/azqueue/azqueue.go | 24 +- .../queue/azqueue/azqueue_suite_test.go | 27 ++ pkg/plugins/queue/azqueue/azqueue_test.go | 373 ++++++++++++++++++ pkg/plugins/queue/azqueue/iface/adapters.go | 31 +- pkg/plugins/queue/azqueue/iface/iface.go | 7 +- 6 files changed, 446 insertions(+), 20 deletions(-) create mode 100644 pkg/plugins/queue/azqueue/azqueue_suite_test.go create mode 100644 pkg/plugins/queue/azqueue/azqueue_test.go diff --git a/makefile b/makefile index 99e60e031..e69da274e 100644 --- a/makefile +++ b/makefile @@ -199,10 +199,12 @@ generate-mocks: @mkdir -p mocks/s3 @mkdir -p mocks/azblob @mkdir -p mocks/mock_event_grid + @mkdir -p mocks/azqueue @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/secret_manager/mock.go @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/storage/azblob/iface AzblobServiceUrlIface,AzblobContainerUrlIface,AzblobBlockBlobUrlIface,AzblobDownloadResponse > mocks/azblob/mock.go @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/key_vault/mock.go @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi BaseClientAPI > mocks/mock_event_grid/mock.go - @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi TopicsClientAPI > mocks/mock_event_grid/topic.go \ No newline at end of file + @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi TopicsClientAPI > mocks/mock_event_grid/topic.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/queue/azqueue/iface AzqueueServiceUrlIface,AzqueueQueueUrlIface,AzqueueMessageUrlIface,AzqueueMessageIdUrlIface,DequeueMessagesResponseIface > mocks/azqueue/mock.go \ No newline at end of file diff --git a/pkg/plugins/queue/azqueue/azqueue.go b/pkg/plugins/queue/azqueue/azqueue.go index 9054a17a0..39ffbde8e 100644 --- a/pkg/plugins/queue/azqueue/azqueue.go +++ b/pkg/plugins/queue/azqueue/azqueue.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package azure_storage_queue_service +package azqueue_service import ( "context" @@ -38,25 +38,25 @@ import ( // Set to 30 seconds, const defaultVisibilityTimeout = 30 * time.Second -type AzureStorageQueueService struct { +type AzqueueQueueService struct { client azqueueserviceiface.AzqueueServiceUrlIface } // Returns an adapted azqueue MessagesUrl, which is a client for interacting with messages in a specific queue -func (s *AzureStorageQueueService) getMessagesUrl(queue string) azqueueserviceiface.AzqueueMessageUrlIface { +func (s *AzqueueQueueService) getMessagesUrl(queue string) azqueueserviceiface.AzqueueMessageUrlIface { qUrl := s.client.NewQueueURL(queue) // Get a new messages URL (used to interact with messages in the queue) return qUrl.NewMessageURL() } // Returns an adapted azqueue MessageIdUrl, which is a client for interacting with a specific message (task) in a specific queue -func (s *AzureStorageQueueService) getMessageIdUrl(queue string, messageId azqueue.MessageID) azqueueserviceiface.AzqueueMessageIdUrlIface { +func (s *AzqueueQueueService) getMessageIdUrl(queue string, messageId azqueue.MessageID) azqueueserviceiface.AzqueueMessageIdUrlIface { mUrl := s.getMessagesUrl(queue) return mUrl.NewMessageIDURL(messageId) } -func (s *AzureStorageQueueService) Send(queue string, task queue.NitricTask) error { +func (s *AzqueueQueueService) Send(queue string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( "AzqueueService.Send", fmt.Sprintf("queue=%s", queue), @@ -85,7 +85,7 @@ func (s *AzureStorageQueueService) Send(queue string, task queue.NitricTask) err return nil } -func (s *AzureStorageQueueService) SendBatch(queueName string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { +func (s *AzqueueQueueService) SendBatch(queueName string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { failedTasks := make([]*queue.FailedTask, 0) for _, task := range tasks { @@ -132,9 +132,9 @@ func leaseFromString(leaseID string) (*AzureQueueItemLease, error) { } // Receive - Receives a collection of tasks off a given queue. -func (s *AzureStorageQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { +func (s *AzqueueQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( - "AzureStorageQueueService.Receive", + "AzqueueQueueService.Receive", fmt.Sprintf("options=%v", options), ) @@ -199,9 +199,9 @@ func (s *AzureStorageQueueService) Receive(options queue.ReceiveOptions) ([]queu } // Complete - Completes a previously popped queue item -func (s *AzureStorageQueueService) Complete(queue string, leaseId string) error { +func (s *AzqueueQueueService) Complete(queue string, leaseId string) error { newErr := errors.ErrorsWithScope( - "AzureStorageQueueService.Complete", + "AzqueueQueueService.Complete", fmt.Sprintf("queue=%s", queue), ) @@ -269,13 +269,13 @@ func New() (queue.QueueService, error) { pipeline := azqueue.NewPipeline(cTkn, azqueue.PipelineOptions{}) client := azqueue.NewServiceURL(*accountURL, pipeline) - return &AzureStorageQueueService{ + return &AzqueueQueueService{ client: azqueueserviceiface.AdaptServiceUrl(client), }, nil } func NewWithClient(client azqueueserviceiface.AzqueueServiceUrlIface) queue.QueueService { - return &AzureStorageQueueService{ + return &AzqueueQueueService{ client: client, } } diff --git a/pkg/plugins/queue/azqueue/azqueue_suite_test.go b/pkg/plugins/queue/azqueue/azqueue_suite_test.go new file mode 100644 index 000000000..16f92c804 --- /dev/null +++ b/pkg/plugins/queue/azqueue/azqueue_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestAzqueue(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Azqueue Suite") +} diff --git a/pkg/plugins/queue/azqueue/azqueue_test.go b/pkg/plugins/queue/azqueue/azqueue_test.go new file mode 100644 index 000000000..00763713e --- /dev/null +++ b/pkg/plugins/queue/azqueue/azqueue_test.go @@ -0,0 +1,373 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service + +import ( + "fmt" + //"bytes" + //"fmt" + "time" + + azqueue2 "github.com/Azure/azure-storage-queue-go/azqueue" + "github.com/nitric-dev/membrane/pkg/plugins/queue" + + //"io/ioutil" + //"strings" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + mock_azqueue "github.com/nitric-dev/membrane/mocks/azqueue" +) + +var _ = Describe("Azqueue", func() { + + Context("Send", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).Times(1).Return(&azqueue2.EnqueueMessageResponse{}, nil) + + err := queuePlugin.Send("test-queue", queue.NitricTask{ + Payload: map[string]interface{}{"testval": "testkey"}, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).Times(1).Return(nil, fmt.Errorf("a test error")) + + err := queuePlugin.Send("test-queue", queue.NitricTask{ + Payload: map[string]interface{}{"testval": "testkey"}, + }) + + By("Returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) + + Context("Send Batch", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(2).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(2).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).Times(2).Return(&azqueue2.EnqueueMessageResponse{}, nil) + + resp, err := queuePlugin.SendBatch("test-queue", []queue.NitricTask{ + {Payload: map[string]interface{}{"testval": "testkey"}}, + {Payload: map[string]interface{}{"testval": "testkey"}}, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Not returning failed tasks") + Expect(len(resp.FailedTasks)).To(Equal(0)) + + crtl.Finish() + }) + }) + + When("Failing to send one task", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(2).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(2).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).AnyTimes( /* Using AnyTimes because Times(2) doesn't work for multiple returns */ + ).Return(nil, fmt.Errorf("a test error")).Return(&azqueue2.EnqueueMessageResponse{}, nil) + + resp, err := queuePlugin.SendBatch("test-queue", []queue.NitricTask{ + {Payload: map[string]interface{}{"testval": "testkey"}}, + {Payload: map[string]interface{}{"testval": "testkey"}}, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Not returning failed tasks") + Expect(resp.FailedTasks).To(Equal([]*queue.FailedTask{})) + + crtl.Finish() + }) + }) + }) + + Context("Receive", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Dequeue once on the Message URL with the expected options") + mockMessages.EXPECT().Dequeue( + gomock.Any(), // ctx + int32(1), // depth + 30*time.Second, // visibility timeout - defaulted to 30 seconds + ).Times(1).Return(mockDequeueResp, nil) + + mockDequeueResp.EXPECT().NumMessages().AnyTimes().Return(int32(1)) + mockDequeueResp.EXPECT().Message(int32(0)).Times(1).Return(&azqueue2.DequeuedMessage{ + ID: "testid", + //InsertionTime: time.Time{}, + //ExpirationTime: time.Time{}, + PopReceipt: "popreceipt", + NextVisibleTime: time.Time{}, + DequeueCount: 0, + Text: "{\"payload\":{\"testval\":\"testkey\"}}", + }) + + depth := uint32(1) + + tasks, err := queuePlugin.Receive(queue.ReceiveOptions{ + QueueName: "test-queue", + Depth: &depth, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Returning the dequeued task") + Expect(len(tasks)).To(Equal(1)) + Expect(tasks[0].Payload).To(Equal(map[string]interface{}{"testval": "testkey"})) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Dequeue once on the Message URL with the expected options") + mockMessages.EXPECT().Dequeue( + gomock.Any(), // ctx + int32(1), // depth + 30*time.Second, // visibility timeout - defaulted to 30 seconds + ).Times(1).Return(nil, fmt.Errorf("a test error")) + + depth := uint32(1) + + _, err := queuePlugin.Receive(queue.ReceiveOptions{ + QueueName: "test-queue", + Depth: &depth, + }) + + By("Returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) + + Context("Complete", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + lease := AzureQueueItemLease{ + ID: "testid", + PopReceipt: "testreceipt", + } + leaseStr, _ := lease.String() + + By("Retrieving the Message ID URL specific to the dequeued task") + mockMessages.EXPECT().NewMessageIDURL(azqueue2.MessageID("testid")).Times(1).Return(mockMessageId) + mockMessageId.EXPECT().Delete(gomock.Any(), azqueue2.PopReceipt(lease.PopReceipt)).Times(1).Return(nil, nil) + + err := queuePlugin.Complete("test-queue", leaseStr) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + lease := AzureQueueItemLease{ + ID: "testid", + PopReceipt: "testreceipt", + } + leaseStr, _ := lease.String() + + By("Retrieving the Message ID URL specific to the dequeued task") + mockMessages.EXPECT().NewMessageIDURL(azqueue2.MessageID("testid")).Times(1).Return(mockMessageId) + mockMessageId.EXPECT().Delete(gomock.Any(), azqueue2.PopReceipt(lease.PopReceipt)).Times(1).Return(nil, fmt.Errorf("a test error")) + + err := queuePlugin.Complete("test-queue", leaseStr) + + By("Returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) +}) diff --git a/pkg/plugins/queue/azqueue/iface/adapters.go b/pkg/plugins/queue/azqueue/iface/adapters.go index 3cae3bf09..fa64bd1b2 100644 --- a/pkg/plugins/queue/azqueue/iface/adapters.go +++ b/pkg/plugins/queue/azqueue/iface/adapters.go @@ -37,11 +37,18 @@ func AdaptMessageIdUrl(c azqueue.MessageIDURL) AzqueueMessageIdUrlIface { return messageIdUrl{c} } +func AdaptDequeueMessagesResponse(c azqueue.DequeuedMessagesResponse) DequeueMessagesResponseIface { + return dequeueMessagesResponse{c} +} + type ( - serviceUrl struct{ c azqueue.ServiceURL } - queueUrl struct{ c azqueue.QueueURL } - messageUrl struct{ c azqueue.MessagesURL } - messageIdUrl struct{ c azqueue.MessageIDURL } + serviceUrl struct{ c azqueue.ServiceURL } + queueUrl struct{ c azqueue.QueueURL } + messageUrl struct{ c azqueue.MessagesURL } + messageIdUrl struct{ c azqueue.MessageIDURL } + dequeueMessagesResponse struct { + c azqueue.DequeuedMessagesResponse + } ) func (c serviceUrl) NewQueueURL(queueName string) AzqueueQueueUrlIface { @@ -56,8 +63,12 @@ func (c messageUrl) Enqueue(ctx context.Context, messageText string, visibilityT return c.c.Enqueue(ctx, messageText, visibilityTimeout, timeToLive) } -func (c messageUrl) Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (*azqueue.DequeuedMessagesResponse, error) { - return c.c.Dequeue(ctx, maxMessages, visibilityTimeout) +func (c messageUrl) Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (DequeueMessagesResponseIface, error) { + resp, err := c.c.Dequeue(ctx, maxMessages, visibilityTimeout) + if err != nil { + return nil, err + } + return AdaptDequeueMessagesResponse(*resp), nil } func (c messageUrl) NewMessageIDURL(messageId azqueue.MessageID) AzqueueMessageIdUrlIface { @@ -67,3 +78,11 @@ func (c messageUrl) NewMessageIDURL(messageId azqueue.MessageID) AzqueueMessageI func (c messageIdUrl) Delete(ctx context.Context, popReceipt azqueue.PopReceipt) (*azqueue.MessageIDDeleteResponse, error) { return c.c.Delete(ctx, popReceipt) } + +func (c dequeueMessagesResponse) NumMessages() int32 { + return c.c.NumMessages() +} + +func (c dequeueMessagesResponse) Message(index int32) *azqueue.DequeuedMessage { + return c.c.Message(index) +} diff --git a/pkg/plugins/queue/azqueue/iface/iface.go b/pkg/plugins/queue/azqueue/iface/iface.go index 2be51ad26..6ce2dfd58 100644 --- a/pkg/plugins/queue/azqueue/iface/iface.go +++ b/pkg/plugins/queue/azqueue/iface/iface.go @@ -31,10 +31,15 @@ type AzqueueQueueUrlIface interface { type AzqueueMessageUrlIface interface { Enqueue(ctx context.Context, messageText string, visibilityTimeout time.Duration, timeToLive time.Duration) (*azqueue.EnqueueMessageResponse, error) - Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (*azqueue.DequeuedMessagesResponse, error) + Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (DequeueMessagesResponseIface, error) NewMessageIDURL(messageId azqueue.MessageID) AzqueueMessageIdUrlIface } type AzqueueMessageIdUrlIface interface { Delete(ctx context.Context, popReceipt azqueue.PopReceipt) (*azqueue.MessageIDDeleteResponse, error) } + +type DequeueMessagesResponseIface interface { + NumMessages() int32 + Message(index int32) *azqueue.DequeuedMessage +} From 92ae04efec5e20954dbf91396931f62e4b4b476b Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 21 Sep 2021 13:56:16 +1000 Subject: [PATCH 59/83] chore: Address review feedback. --- go.mod | 10 +-- go.sum | 84 +---------------------- pkg/plugins/queue/azqueue/azqueue.go | 23 ++++--- pkg/plugins/queue/azqueue/azqueue_test.go | 5 -- pkg/plugins/storage/azblob/azblob.go | 27 +++++--- pkg/providers/azure/utils/const.go | 7 +- 6 files changed, 45 insertions(+), 111 deletions(-) diff --git a/go.mod b/go.mod index 665599f39..b0c41ae8d 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,14 @@ require ( cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.10.0 github.com/Azure/azure-sdk-for-go v56.3.0+incompatible + github.com/Azure/azure-storage-blob-go v0.14.0 + github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd github.com/Azure/go-autorest/autorest v0.11.18 - github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect - github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.14 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/DataDog/zstd v1.4.8 // indirect github.com/Knetic/govaluate v3.0.0+incompatible diff --git a/go.sum b/go.sum index 0afbccf12..dbad3bd2f 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,8 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.5.0 h1:4qNItsmc4GP6UOZPGemmHY4ZfPofVhcaKXsYw9wm9oA= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= @@ -40,7 +38,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= @@ -58,11 +55,9 @@ github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1E github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.14 h1:G8hexQdV5D4khOXrWG2YuLCFKhWYmWD8bHYaXN5ophk= github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.15 h1:X+p2GF0GWyOiSmqohIaEeuNFNDY4I4EOlVuUQvFdWMk= -github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= @@ -81,9 +76,7 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -101,19 +94,13 @@ github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -128,62 +115,46 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= github.com/envoyproxy/protoc-gen-validate v0.6.1/go.mod h1:txg5va2Qkip90uYoSKH+nkAAmXrb2j3iq4FLwdrCbXQ= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3 h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -220,7 +191,6 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= github.com/google/addlicense v1.0.0/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -249,9 +219,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2 h1:LR89qFljJ48s990kEKGsk213yIJDPI4205OKOzbURK8= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= @@ -264,44 +232,33 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7 h1:ux/56T2xqZO/3cP1I2F86qpeoYPCOzk+KF/UH/Ar+lk= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -309,9 +266,7 @@ github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tW github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-star v0.5.1 h1:sImehRT+p7lW9n6R7MQc5hVgzWGEkDVZU4AsBQ4Isu8= github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= @@ -323,7 +278,6 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/muesli/termenv v0.7.4 h1:/pBqvU5CpkY53tU0vVn+xgs2ZTX63aH5nY+SSps5Xa8= github.com/muesli/termenv v0.7.4/go.mod h1:pZ7qY9l3F7e5xsAOS0zCew2tME+p7bWeBkotCEcIIcc= @@ -337,23 +291,17 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= -github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -363,19 +311,13 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.3.4 h1:8q6vk3hthlpb2SouZcnBVKboxWQWMDNF38bwholZrJc= github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -394,7 +336,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -410,9 +351,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0 h1:OtISOGfH6sOWa1/qXqqAiOIAO6Z5J3AEAE18WAq6BiQ= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -447,10 +385,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -466,7 +402,6 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -514,8 +449,6 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -589,14 +522,10 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210223095934-7937bea0104d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -610,7 +539,6 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -665,8 +593,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6 h1:SIasE1FVIQOWz2GEAHFOmoW7xchJcqlucjSULTL0Ag4= golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -776,9 +702,7 @@ gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -798,11 +722,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/plugins/queue/azqueue/azqueue.go b/pkg/plugins/queue/azqueue/azqueue.go index 39ffbde8e..557a5b755 100644 --- a/pkg/plugins/queue/azqueue/azqueue.go +++ b/pkg/plugins/queue/azqueue/azqueue.go @@ -58,8 +58,10 @@ func (s *AzqueueQueueService) getMessageIdUrl(queue string, messageId azqueue.Me func (s *AzqueueQueueService) Send(queue string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( - "AzqueueService.Send", - fmt.Sprintf("queue=%s", queue), + "AzqueueQueueService.Send", + map[string]interface{}{ + "queue": queue, + }, ) messages := s.getMessagesUrl(queue) @@ -135,7 +137,9 @@ func leaseFromString(leaseID string) (*AzureQueueItemLease, error) { func (s *AzqueueQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( "AzqueueQueueService.Receive", - fmt.Sprintf("options=%v", options), + map[string]interface{}{ + "options": options, + }, ) if err := options.Validate(); err != nil { @@ -202,7 +206,10 @@ func (s *AzqueueQueueService) Receive(options queue.ReceiveOptions) ([]queue.Nit func (s *AzqueueQueueService) Complete(queue string, leaseId string) error { newErr := errors.ErrorsWithScope( "AzqueueQueueService.Complete", - fmt.Sprintf("queue=%s", queue), + map[string]interface{}{ + "queue": queue, + "leaseId": leaseId, + }, ) lease, err := leaseFromString(leaseId) @@ -249,9 +256,9 @@ func tokenRefresherFromSpt(spt *adal.ServicePrincipalToken) azqueue.TokenRefresh // New - Constructs a new Azure Storage Queues client with defaults func New() (queue.QueueService, error) { - storageAccountName := utils.GetEnv(azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV, "") - if storageAccountName == "" { - return nil, fmt.Errorf("failed to determine Azure Storage Account Name, environment variable %s not set", azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV) + queueUrl := utils.GetEnv(azureutils.AZURE_STORAGE_QUEUE_ENDPOINT, "") + if queueUrl == "" { + return nil, fmt.Errorf("failed to determine Azure Storage Queue endpoint, environment variable %s not set", azureutils.AZURE_STORAGE_QUEUE_ENDPOINT) } spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.Storage) @@ -262,7 +269,7 @@ func New() (queue.QueueService, error) { cTkn := azqueue.NewTokenCredential(spt.Token().AccessToken, tokenRefresherFromSpt(spt)) var accountURL *url.URL - if accountURL, err = url.Parse(fmt.Sprintf("https://%s.queue.core.windows.net", storageAccountName)); err != nil { + if accountURL, err = url.Parse(queueUrl); err != nil { return nil, err } diff --git a/pkg/plugins/queue/azqueue/azqueue_test.go b/pkg/plugins/queue/azqueue/azqueue_test.go index 00763713e..36d6a7ca3 100644 --- a/pkg/plugins/queue/azqueue/azqueue_test.go +++ b/pkg/plugins/queue/azqueue/azqueue_test.go @@ -16,16 +16,11 @@ package azqueue_service import ( "fmt" - //"bytes" - //"fmt" "time" azqueue2 "github.com/Azure/azure-storage-queue-go/azqueue" "github.com/nitric-dev/membrane/pkg/plugins/queue" - //"io/ioutil" - //"strings" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/plugins/storage/azblob/azblob.go b/pkg/plugins/storage/azblob/azblob.go index a53e6b4e3..a1e5bc48d 100644 --- a/pkg/plugins/storage/azblob/azblob.go +++ b/pkg/plugins/storage/azblob/azblob.go @@ -36,6 +36,7 @@ import ( // AzblobStorageService - Nitric membrane storage plugin implementation for Azure Storage type AzblobStorageService struct { client azblob_service_iface.AzblobServiceUrlIface + storage.UnimplementedStoragePlugin } func (a *AzblobStorageService) getBlobUrl(bucket string, key string) azblob_service_iface.AzblobBlockBlobUrlIface { @@ -47,8 +48,10 @@ func (a *AzblobStorageService) getBlobUrl(bucket string, key string) azblob_serv func (a *AzblobStorageService) Read(bucket string, key string) ([]byte, error) { newErr := errors.ErrorsWithScope( "AzblobStorageService.Read", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) // Get the bucket for this bucket name blob := a.getBlobUrl(bucket, key) @@ -79,8 +82,10 @@ func (a *AzblobStorageService) Read(bucket string, key string) ([]byte, error) { func (a *AzblobStorageService) Write(bucket string, key string, object []byte) error { newErr := errors.ErrorsWithScope( "AzblobStorageService.Write", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) blob := a.getBlobUrl(bucket, key) @@ -108,8 +113,10 @@ func (a *AzblobStorageService) Write(bucket string, key string, object []byte) e func (a *AzblobStorageService) Delete(bucket string, key string) error { newErr := errors.ErrorsWithScope( "AzblobStorageService.Delete", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) // Get the bucket for this bucket name @@ -153,9 +160,9 @@ func New() (storage.StorageService, error) { // TODO: Create a default storage account for the stack??? // XXX: This will limit a membrane wrapped application // to accessing a single storage account - storageAccountName := utils.GetEnv(azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV, "") - if storageAccountName == "" { - return nil, fmt.Errorf("failed to determine Azure Storage Account Name, environment variable %s not set", azureutils.AZURE_STORAGE_ACCOUNT_NAME_ENV) + blobEndpoint := utils.GetEnv(azureutils.AZURE_STORAGE_BLOB_ENDPOINT, "") + if blobEndpoint == "" { + return nil, fmt.Errorf("failed to determine Azure Storage Blob endpoint, environment variable %s not set", azureutils.AZURE_STORAGE_BLOB_ENDPOINT) } spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.Storage) @@ -166,7 +173,7 @@ func New() (storage.StorageService, error) { cTkn := azblob.NewTokenCredential(spt.Token().AccessToken, tokenRefresherFromSpt(spt)) var accountURL *url.URL - if accountURL, err = url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", storageAccountName)); err != nil { + if accountURL, err = url.Parse(blobEndpoint); err != nil { return nil, err } diff --git a/pkg/providers/azure/utils/const.go b/pkg/providers/azure/utils/const.go index 9c0d3210b..26f52f8ad 100644 --- a/pkg/providers/azure/utils/const.go +++ b/pkg/providers/azure/utils/const.go @@ -14,5 +14,8 @@ package utils -// AZURE_STORAGE_ACCOUNT_NAME_ENV - Default Azure Storage Account Name for Nitric Stack -const AZURE_STORAGE_ACCOUNT_NAME_ENV = "AZURE_STORAGE_ACCOUNT_NAME" +// AZURE_STORAGE_BLOB_ENDPOINT - Endpoint for azblob storage plugin +const AZURE_STORAGE_BLOB_ENDPOINT = "AZURE_STORAGE_ACCOUNT_BLOB_ENDPOINT" + +// AZURE_STORAGE_BLOB_ENDPOINT - Endpoint for azqueue queue plugin +const AZURE_STORAGE_QUEUE_ENDPOINT = "AZURE_STORAGE_ACCOUNT_QUEUE_ENDPOINT" From 41b7c13f1b0d24a56fe82bfae9ebf65bf46a200d Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 16:08:17 +1000 Subject: [PATCH 60/83] feat(documents/mongodb): Add document streaming support. --- pkg/plugins/document/mongodb/mongodb.go | 188 ++++++++++++++++-------- 1 file changed, 125 insertions(+), 63 deletions(-) diff --git a/pkg/plugins/document/mongodb/mongodb.go b/pkg/plugins/document/mongodb/mongodb.go index 5211b0412..6b68b94b5 100644 --- a/pkg/plugins/document/mongodb/mongodb.go +++ b/pkg/plugins/document/mongodb/mongodb.go @@ -17,6 +17,7 @@ package mongodb_service import ( "context" "fmt" + "io" "strings" "time" @@ -237,32 +238,7 @@ func (s *MongoDocService) Delete(key *document.Key) error { return nil } -func (s *MongoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { - newErr := errors.ErrorsWithScope( - "MongoDocService.Query", - map[string]interface{}{ - "collection": collection, - }, - ) - - if err := document.ValidateQueryCollection(collection); err != nil { - return nil, newErr( - codes.InvalidArgument, - "invalid key", - err, - ) - } - - if err := document.ValidateExpressions(expressions); err != nil { - return nil, newErr( - codes.InvalidArgument, - "invalid expressions", - err, - ) - } - - var orderByAttrib string - +func (s *MongoDocService) getCursor(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (cursor *mongo.Cursor, orderBy string, err error) { coll := s.getCollection(&document.Key{Collection: collection}) query := bson.M{} @@ -311,17 +287,46 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d } } - if exp.Operator != "==" && limit > 0 && orderByAttrib == "" { + if exp.Operator != "==" && limit > 0 && orderBy == "" { opts.SetSort(bson.D{{expOperand, 1}}) - orderByAttrib = expOperand + orderBy = expOperand } } + cursor, err = coll.Find(s.context, query, opts) + + return +} + +func (s *MongoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { + newErr := errors.ErrorsWithScope( + "MongoDocService.Query", + map[string]interface{}{ + "collection": collection, + }, + ) + + if err := document.ValidateQueryCollection(collection); err != nil { + return nil, newErr( + codes.InvalidArgument, + "invalid key", + err, + ) + } + + if err := document.ValidateExpressions(expressions); err != nil { + return nil, newErr( + codes.InvalidArgument, + "invalid expressions", + err, + ) + } + queryResult := &document.QueryResult{ Documents: make([]document.Document, 0), } - cursor, err := coll.Find(s.context, query, opts) + cursor, orderBy, err := s.getCursor(collection, expressions, limit, pagingToken) if err != nil { return nil, newErr( @@ -333,62 +338,119 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d defer cursor.Close(s.context) for cursor.Next(s.context) { - var docSnap map[string]interface{} - - err := cursor.Decode(&docSnap) + sdkDoc, err := mongoDocToDocument(collection, cursor) if err != nil { return nil, newErr( codes.Internal, - "error querying value", + "error decoding mongo document", err, ) } - id := docSnap[primaryKeyAttr].(string) + queryResult.Documents = append(queryResult.Documents, *sdkDoc) - // remove id from content - delete(docSnap, primaryKeyAttr) + // If query limit configured determine continue tokens + if limit > 0 && len(queryResult.Documents) == limit { + tokens := "" + if orderBy != "" { + tokens = fmt.Sprintf("%v", sdkDoc.Content[orderBy]) + "|" + } + tokens += sdkDoc.Key.Id - sdkDoc := document.Document{ - Content: docSnap, - Key: &document.Key{ - Collection: collection, - Id: id, - }, + queryResult.PagingToken = map[string]string{ + "pagingTokens": tokens, + } } + } - if docSnap[parentKeyAttr] != nil { - parentId := docSnap[parentKeyAttr].(string) + return queryResult, nil +} - sdkDoc.Key.Collection = &document.Collection{ - Name: collection.Name, - Parent: &document.Key{ - Collection: collection.Parent.Collection, - Id: parentId, - }, - } +func (s *MongoDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "MongoDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) + + colErr := document.ValidateQueryCollection(collection) + expErr := document.ValidateExpressions(expressions) - delete(docSnap, parentKeyAttr) + if colErr != nil || expErr != nil { + // Return an error only iterator + return func() (*document.Document, error) { + return nil, newErr( + codes.InvalidArgument, + "invalid arguments", + fmt.Errorf("collection error:%v, expression error: %v", colErr, expErr), + ) } + } - queryResult.Documents = append(queryResult.Documents, sdkDoc) + cursor, _, cursorErr := s.getCursor(collection, expressions, limit, nil) - // If query limit configured determine continue tokens - if limit > 0 && len(queryResult.Documents) == limit { - tokens := "" - if orderByAttrib != "" { - tokens = fmt.Sprintf("%v", docSnap[orderByAttrib]) + "|" - } - tokens += id + return func() (*document.Document, error) { + if cursorErr != nil { + return nil, cursorErr + } - queryResult.PagingToken = map[string]string{ - "pagingTokens": tokens, + if cursor.Next(s.context) { + // return the next document + return mongoDocToDocument(collection, cursor) + } else { + // there was an error + // Close the cursor + cursor.Close(s.context) + + // Examine the cursors error to see if it's exhausted + if cursor.Err() != nil { + return nil, cursor.Err() + } else { + return nil, io.EOF } } } +} - return queryResult, nil +func mongoDocToDocument(coll *document.Collection, cursor *mongo.Cursor) (*document.Document, error) { + var docSnap map[string]interface{} + + err := cursor.Decode(&docSnap) + + if err != nil { + return nil, fmt.Errorf("error decoding mongo document") + } + + id := docSnap[primaryKeyAttr].(string) + + // remove id from content + delete(docSnap, primaryKeyAttr) + + sdkDoc := document.Document{ + Content: docSnap, + Key: &document.Key{ + Collection: coll, + Id: id, + }, + } + + if docSnap[parentKeyAttr] != nil { + parentId := docSnap[parentKeyAttr].(string) + + sdkDoc.Key.Collection = &document.Collection{ + Name: coll.Name, + Parent: &document.Key{ + Collection: coll.Parent.Collection, + Id: parentId, + }, + } + + delete(docSnap, parentKeyAttr) + } + + return &sdkDoc, nil } func New() (document.DocumentService, error) { From 71d9805acc6fa3d1ca259a707691daa331a9d0de Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 17 Sep 2021 16:08:39 +1000 Subject: [PATCH 61/83] test(documents/mongodb): Connect query stream test harness. --- tests/plugins/document/mongodb/mongodb_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/plugins/document/mongodb/mongodb_test.go b/tests/plugins/document/mongodb/mongodb_test.go index 93337a6d4..1fb51afef 100644 --- a/tests/plugins/document/mongodb/mongodb_test.go +++ b/tests/plugins/document/mongodb/mongodb_test.go @@ -88,4 +88,5 @@ var _ = Describe("MongoDB", func() { test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) From 17aac74629cc36786077cf51c977d4ab603b79a1 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Wed, 22 Sep 2021 09:01:24 +1000 Subject: [PATCH 62/83] refactor: compact validation to reduce duplication. --- pkg/plugins/document/mongodb/mongodb.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pkg/plugins/document/mongodb/mongodb.go b/pkg/plugins/document/mongodb/mongodb.go index 6b68b94b5..cb506e19f 100644 --- a/pkg/plugins/document/mongodb/mongodb.go +++ b/pkg/plugins/document/mongodb/mongodb.go @@ -306,19 +306,11 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d }, ) - if err := document.ValidateQueryCollection(collection); err != nil { + if colErr, expErr := document.ValidateQueryCollection(collection), document.ValidateExpressions(expressions); colErr != nil || expErr != nil { return nil, newErr( codes.InvalidArgument, - "invalid key", - err, - ) - } - - if err := document.ValidateExpressions(expressions); err != nil { - return nil, newErr( - codes.InvalidArgument, - "invalid expressions", - err, + "invalid arguments", + fmt.Errorf("collection: %v, expressions%v", colErr, expErr), ) } From 931e9a054a9864cc2ed99afb5952ad2570aecc76 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Wed, 22 Sep 2021 13:49:56 +1000 Subject: [PATCH 63/83] chore: Error wrapping updates. --- pkg/plugins/document/mongodb/mongodb.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/plugins/document/mongodb/mongodb.go b/pkg/plugins/document/mongodb/mongodb.go index cb506e19f..f09f4a38c 100644 --- a/pkg/plugins/document/mongodb/mongodb.go +++ b/pkg/plugins/document/mongodb/mongodb.go @@ -390,15 +390,28 @@ func (s *MongoDocService) QueryStream(collection *document.Collection, expressio if cursor.Next(s.context) { // return the next document - return mongoDocToDocument(collection, cursor) + doc, err := mongoDocToDocument(collection, cursor) + + if err != nil { + return nil, newErr( + codes.Internal, + "error decoding mongo document", + err, + ) + } + + return doc, nil } else { // there was an error // Close the cursor cursor.Close(s.context) - // Examine the cursors error to see if it's exhausted if cursor.Err() != nil { - return nil, cursor.Err() + return nil, newErr( + codes.Internal, + "mongo cursor error", + cursor.Err(), + ) } else { return nil, io.EOF } @@ -409,10 +422,8 @@ func (s *MongoDocService) QueryStream(collection *document.Collection, expressio func mongoDocToDocument(coll *document.Collection, cursor *mongo.Cursor) (*document.Document, error) { var docSnap map[string]interface{} - err := cursor.Decode(&docSnap) - - if err != nil { - return nil, fmt.Errorf("error decoding mongo document") + if err := cursor.Decode(&docSnap); err != nil { + return nil, err } id := docSnap[primaryKeyAttr].(string) From fbb80feaef7ce34d059ee4936b56594cea97b1c6 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Wed, 22 Sep 2021 13:52:24 +1000 Subject: [PATCH 64/83] chore: Add additional scope logging. --- pkg/plugins/queue/azqueue/azqueue.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/plugins/queue/azqueue/azqueue.go b/pkg/plugins/queue/azqueue/azqueue.go index 557a5b755..63bfcd613 100644 --- a/pkg/plugins/queue/azqueue/azqueue.go +++ b/pkg/plugins/queue/azqueue/azqueue.go @@ -61,6 +61,7 @@ func (s *AzqueueQueueService) Send(queue string, task queue.NitricTask) error { "AzqueueQueueService.Send", map[string]interface{}{ "queue": queue, + "task": task, }, ) From 2d29a8cde8013efad0e36570baae067a25b0df64 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Thu, 30 Sep 2021 14:58:30 +1000 Subject: [PATCH 65/83] feat(azblob): Add PresignUrl method. --- pkg/plugins/storage/azblob/azblob.go | 49 ++++++++++++++++++++ pkg/plugins/storage/azblob/iface/adapters.go | 4 ++ pkg/plugins/storage/azblob/iface/iface.go | 1 + 3 files changed, 54 insertions(+) diff --git a/pkg/plugins/storage/azblob/azblob.go b/pkg/plugins/storage/azblob/azblob.go index a1e5bc48d..7df9297ae 100644 --- a/pkg/plugins/storage/azblob/azblob.go +++ b/pkg/plugins/storage/azblob/azblob.go @@ -137,6 +137,55 @@ func (a *AzblobStorageService) Delete(bucket string, key string) error { return nil } +func (s *AzblobStorageService) PreSignUrl(bucket string, key string, operation storage.Operation, expiry uint32) (string, error) { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.PreSignUrl", + map[string]interface{}{ + "bucket": bucket, + "key": key, + "operation": operation.String(), + }, + ) + + blobUrlParts := azblob.NewBlobURLParts(s.getBlobUrl(bucket, key)) + validDuration := time.Now().UTC().Add(expiry * uint32(time.Second)) + cred, err := s.client.GetUserDelegationCredential(context.TODO(), azblob.NewKeyInfo(currentTime, validDuration, nil, nil)) + + if err != nil { + return "", newErr( + codes.Internal, + "could not get user delegation credential", + err, + ) + } + + sigOpts := azblob.BlobSASSignatureValues{ + Protocol: azblob.SASProtocolHTTPS, + ExpiryTime: time.Now().UTC().Add(expiry * uint32(time.Second)), + Permissions: azblob.BlobSASPermissions{ + Read: operation == storage.READ, + Write: operation == storage.WRITE, + }.String(), + BlobName: key, + ContainerName: bucket, + } + + queryParams, err := sigOpts.NewSASQueryParameters(cred) + + if err != nil { + return "", newErr( + codes.Internal, + "error signing query params for URL", + err, + ) + } + + blobUrlParts.SAS = queryParams + url := blobUrlParts.URL() + + return url.String(), nil +} + const expiryBuffer = 2 * time.Minute func tokenRefresherFromSpt(spt *adal.ServicePrincipalToken) azblob.TokenRefresher { diff --git a/pkg/plugins/storage/azblob/iface/adapters.go b/pkg/plugins/storage/azblob/iface/adapters.go index 0318cbdb6..596415c49 100644 --- a/pkg/plugins/storage/azblob/iface/adapters.go +++ b/pkg/plugins/storage/azblob/iface/adapters.go @@ -43,6 +43,10 @@ func (c serviceUrl) NewContainerURL(bucket string) AzblobContainerUrlIface { return AdaptContainerUrl(c.c.NewContainerURL(bucket)) } +func (c serviceUrl) GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.UserDelegationCredential, error) { + return c.c.GetUserDelegationCredential(ctx, info, timeout, requestID) +} + func (c containerUrl) NewBlockBlobURL(blob string) AzblobBlockBlobUrlIface { return AdaptBlobUrl(c.c.NewBlockBlobURL(blob)) } diff --git a/pkg/plugins/storage/azblob/iface/iface.go b/pkg/plugins/storage/azblob/iface/iface.go index 37f0fa65d..303838937 100644 --- a/pkg/plugins/storage/azblob/iface/iface.go +++ b/pkg/plugins/storage/azblob/iface/iface.go @@ -25,6 +25,7 @@ import ( // for azblob.ServiceUrl type AzblobServiceUrlIface interface { NewContainerURL(string) AzblobContainerUrlIface + GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.UserDelegationCredential, error) } // AzblobContainerUrlIface - Mockable client interface From 90f16b91f04b4270b3f8ff194fa2ae4e9824a682 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Thu, 30 Sep 2021 16:00:57 +1000 Subject: [PATCH 66/83] chore: fix compile errors and add WIP tests. --- go.mod | 2 +- go.sum | 2 + pkg/plugins/storage/azblob/azblob.go | 9 ++- pkg/plugins/storage/azblob/azblob_test.go | 81 ++++++++++++++++++++ pkg/plugins/storage/azblob/iface/adapters.go | 7 +- pkg/plugins/storage/azblob/iface/iface.go | 4 +- 6 files changed, 98 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index b0c41ae8d..b44fa8329 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.6 + golang.org/x/tools v0.1.7 google.golang.org/api v0.40.0 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c google.golang.org/grpc v1.35.0 diff --git a/go.sum b/go.sum index dbad3bd2f..ebe9c52a3 100644 --- a/go.sum +++ b/go.sum @@ -595,6 +595,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.6 h1:SIasE1FVIQOWz2GEAHFOmoW7xchJcqlucjSULTL0Ag4= golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/plugins/storage/azblob/azblob.go b/pkg/plugins/storage/azblob/azblob.go index 7df9297ae..849c05d75 100644 --- a/pkg/plugins/storage/azblob/azblob.go +++ b/pkg/plugins/storage/azblob/azblob.go @@ -147,9 +147,10 @@ func (s *AzblobStorageService) PreSignUrl(bucket string, key string, operation s }, ) - blobUrlParts := azblob.NewBlobURLParts(s.getBlobUrl(bucket, key)) - validDuration := time.Now().UTC().Add(expiry * uint32(time.Second)) - cred, err := s.client.GetUserDelegationCredential(context.TODO(), azblob.NewKeyInfo(currentTime, validDuration, nil, nil)) + blobUrlParts := azblob.NewBlobURLParts(s.getBlobUrl(bucket, key).Url()) + currentTime := time.Now().UTC() + validDuration := currentTime.Add(time.Duration(expiry) * time.Second) + cred, err := s.client.GetUserDelegationCredential(context.TODO(), azblob.NewKeyInfo(currentTime, validDuration), nil, nil) if err != nil { return "", newErr( @@ -161,7 +162,7 @@ func (s *AzblobStorageService) PreSignUrl(bucket string, key string, operation s sigOpts := azblob.BlobSASSignatureValues{ Protocol: azblob.SASProtocolHTTPS, - ExpiryTime: time.Now().UTC().Add(expiry * uint32(time.Second)), + ExpiryTime: validDuration, Permissions: azblob.BlobSASPermissions{ Read: operation == storage.READ, Write: operation == storage.WRITE, diff --git a/pkg/plugins/storage/azblob/azblob_test.go b/pkg/plugins/storage/azblob/azblob_test.go index 58fc99b1e..6c6529ddb 100644 --- a/pkg/plugins/storage/azblob/azblob_test.go +++ b/pkg/plugins/storage/azblob/azblob_test.go @@ -16,8 +16,10 @@ package azblob_service import ( "bytes" + "context" "fmt" "io/ioutil" + "net/url" "strings" "github.com/Azure/azure-storage-blob-go/azblob" @@ -26,8 +28,31 @@ import ( . "github.com/onsi/gomega" mock_azblob "github.com/nitric-dev/membrane/mocks/azblob" + "github.com/nitric-dev/membrane/pkg/plugins/storage" ) +type mockStorageCredential struct { + azblob.StorageAccountCredential +} + +//AccountName() string +// ComputeHMACSHA256(message string) (base64String string) +// getUDKParams() *UserDelegationKey + +func (m *mockStorageCredential) AccountName() string { + return "mock-account-name" +} + +func (m *mockStorageCredential) ComputeHMACSHA256(message string) (base64String string) { + base64String = "mock-string" + + return +} + +func (m *mockStorageCredential) getUDKParams() *azblob.UserDelegationKey { + return &azblob.UserDelegationKey{} +} + var _ = Describe("Azblob", func() { //Context("New", func() { // When("", func() { @@ -259,4 +284,60 @@ var _ = Describe("Azblob", func() { }) }) }) + + Context("PresignUrl", func() { + When("User delegation credentials are accessible", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should return a presigned url", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Retrieving user delegation credentials") + mockAzblob.EXPECT().GetUserDelegationCredential( + context.TODO(), gomock.Any(), gomock.Any(), nil, + ).Return( + &mockStorageCredential{}, + nil, + ) + + u, _ := url.Parse("https://fake-account.com/my-bucket/my-blob") + By("Getting the URL") + mockBlob.EXPECT().Url().Return(*u) + + url, err := storagePlugin.PreSignUrl("my-bucket", "my-blob", storage.READ, 3600) + + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + + By("Returning a pre-signed URL") + Expect(url).ToNot(Equal("")) + }) + }) + + //When("retrieving user delegation credentials fails", func() { + // crtl := gomock.NewController(GinkgoT()) + // mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + // mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + // mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + // storagePlugin := &AzblobStorageService{ + // client: mockAzblob, + // } + + // It("should return an error", func() { + + // }) + //}) + }) }) diff --git a/pkg/plugins/storage/azblob/iface/adapters.go b/pkg/plugins/storage/azblob/iface/adapters.go index 596415c49..3eba5451b 100644 --- a/pkg/plugins/storage/azblob/iface/adapters.go +++ b/pkg/plugins/storage/azblob/iface/adapters.go @@ -17,6 +17,7 @@ package azblob_service_iface import ( "context" "io" + "net/url" "github.com/Azure/azure-storage-blob-go/azblob" ) @@ -43,7 +44,7 @@ func (c serviceUrl) NewContainerURL(bucket string) AzblobContainerUrlIface { return AdaptContainerUrl(c.c.NewContainerURL(bucket)) } -func (c serviceUrl) GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.UserDelegationCredential, error) { +func (c serviceUrl) GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.StorageAccountCredential, error) { return c.c.GetUserDelegationCredential(ctx, info, timeout, requestID) } @@ -55,6 +56,10 @@ func (c blobUrl) Download(ctx context.Context, offset int64, count int64, bac az return c.c.Download(ctx, offset, count, bac, f, cpk) } +func (c blobUrl) Url() url.URL { + return c.c.URL() +} + func (c blobUrl) Upload(ctx context.Context, r io.ReadSeeker, h azblob.BlobHTTPHeaders, m azblob.Metadata, bac azblob.BlobAccessConditions, att azblob.AccessTierType, btm azblob.BlobTagsMap, cpk azblob.ClientProvidedKeyOptions) (*azblob.BlockBlobUploadResponse, error) { return c.c.Upload(ctx, r, h, m, bac, att, btm, cpk) } diff --git a/pkg/plugins/storage/azblob/iface/iface.go b/pkg/plugins/storage/azblob/iface/iface.go index 303838937..caced691f 100644 --- a/pkg/plugins/storage/azblob/iface/iface.go +++ b/pkg/plugins/storage/azblob/iface/iface.go @@ -17,6 +17,7 @@ package azblob_service_iface import ( "context" "io" + "net/url" "github.com/Azure/azure-storage-blob-go/azblob" ) @@ -25,7 +26,7 @@ import ( // for azblob.ServiceUrl type AzblobServiceUrlIface interface { NewContainerURL(string) AzblobContainerUrlIface - GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.UserDelegationCredential, error) + GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.StorageAccountCredential, error) } // AzblobContainerUrlIface - Mockable client interface @@ -37,6 +38,7 @@ type AzblobContainerUrlIface interface { // AzblobBlockBlobUrlIface - Mockable client interface // for azblob.BlockBlobUrl type AzblobBlockBlobUrlIface interface { + Url() url.URL Download(context.Context, int64, int64, azblob.BlobAccessConditions, bool, azblob.ClientProvidedKeyOptions) (AzblobDownloadResponse, error) Upload(context.Context, io.ReadSeeker, azblob.BlobHTTPHeaders, azblob.Metadata, azblob.BlobAccessConditions, azblob.AccessTierType, azblob.BlobTagsMap, azblob.ClientProvidedKeyOptions) (*azblob.BlockBlobUploadResponse, error) Delete(context.Context, azblob.DeleteSnapshotsOptionType, azblob.BlobAccessConditions) (*azblob.BlobDeleteResponse, error) From 326f735baa366c7937478c6b7e417a46fa633e26 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 5 Oct 2021 10:12:10 +1100 Subject: [PATCH 67/83] chore: rollback azure storage sdk version to allow presigned urls. --- go.mod | 2 +- go.sum | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b44fa8329..8b5a68509 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.10.0 github.com/Azure/azure-sdk-for-go v56.3.0+incompatible - github.com/Azure/azure-storage-blob-go v0.14.0 + github.com/Azure/azure-storage-blob-go v0.13.0 github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd github.com/Azure/go-autorest/autorest v0.11.18 github.com/Azure/go-autorest/autorest/adal v0.9.14 diff --git a/go.sum b/go.sum index ebe9c52a3..446e02f60 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVt github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= -github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc= +github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -53,6 +53,7 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= @@ -108,6 +109,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= @@ -221,6 +223,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -257,8 +260,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -281,6 +282,8 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/muesli/termenv v0.7.4 h1:/pBqvU5CpkY53tU0vVn+xgs2ZTX63aH5nY+SSps5Xa8= github.com/muesli/termenv v0.7.4/go.mod h1:pZ7qY9l3F7e5xsAOS0zCew2tME+p7bWeBkotCEcIIcc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -593,8 +596,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.6 h1:SIasE1FVIQOWz2GEAHFOmoW7xchJcqlucjSULTL0Ag4= -golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -702,8 +703,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= From dae83242b469c34c9ee999272ecefc8766940ecd Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 5 Oct 2021 10:28:00 +1100 Subject: [PATCH 68/83] test: Add presign url tests. --- pkg/plugins/storage/azblob/azblob_test.go | 52 +++++++++++++++++------ 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/pkg/plugins/storage/azblob/azblob_test.go b/pkg/plugins/storage/azblob/azblob_test.go index 6c6529ddb..4486e9928 100644 --- a/pkg/plugins/storage/azblob/azblob_test.go +++ b/pkg/plugins/storage/azblob/azblob_test.go @@ -307,7 +307,7 @@ var _ = Describe("Azblob", func() { mockAzblob.EXPECT().GetUserDelegationCredential( context.TODO(), gomock.Any(), gomock.Any(), nil, ).Return( - &mockStorageCredential{}, + azblob.NewUserDelegationCredential("mock-account-name", azblob.UserDelegationKey{}), nil, ) @@ -320,24 +320,48 @@ var _ = Describe("Azblob", func() { By("Not returning an error") Expect(err).ShouldNot(HaveOccurred()) - By("Returning a pre-signed URL") - Expect(url).ToNot(Equal("")) + By("Returning a pre-signed URL from the computed blob URL") + Expect(url).To(ContainSubstring("https://fake-account.com/my-bucket/my-blob")) }) }) - //When("retrieving user delegation credentials fails", func() { - // crtl := gomock.NewController(GinkgoT()) - // mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) - // mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) - // mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + When("retrieving user delegation credentials fails", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) - // storagePlugin := &AzblobStorageService{ - // client: mockAzblob, - // } + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } - // It("should return an error", func() { + It("should return an error", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) - // }) - //}) + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Failing to retrieve user delegation credentials") + mockAzblob.EXPECT().GetUserDelegationCredential( + context.TODO(), gomock.Any(), gomock.Any(), nil, + ).Return( + nil, + fmt.Errorf("mock-error"), + ) + + u, _ := url.Parse("https://fake-account.com/my-bucket/my-blob") + By("Getting the URL") + mockBlob.EXPECT().Url().Return(*u) + + url, err := storagePlugin.PreSignUrl("my-bucket", "my-blob", storage.READ, 3600) + + By("Not returning a url") + Expect(url).To(Equal("")) + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) }) }) From 383dde31b183bb4480b920370e6af9727071189a Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 5 Oct 2021 12:05:56 +1100 Subject: [PATCH 69/83] fix: Add support for multiple query params. --- contracts/proto/faas/v1/faas.proto | 12 ++++++++++-- go.mod | 2 +- go.sum | 4 ++-- pkg/triggers/http_request.go | 14 +++++++++----- pkg/worker/faas_http_worker.go | 9 ++++++++- pkg/worker/faas_worker.go | 29 +++++++++++++++++++++-------- pkg/worker/http_worker.go | 4 +++- 7 files changed, 54 insertions(+), 20 deletions(-) diff --git a/contracts/proto/faas/v1/faas.proto b/contracts/proto/faas/v1/faas.proto index ed5d69927..2bb4fc4e9 100644 --- a/contracts/proto/faas/v1/faas.proto +++ b/contracts/proto/faas/v1/faas.proto @@ -74,6 +74,10 @@ message HeaderValue { repeated string value = 1; } +message QueryValue { + repeated string value = 1; +} + message HttpTriggerContext { // The request method @@ -86,11 +90,15 @@ message HttpTriggerContext { // TODO: Remove in 1.0 map headers_old = 3 [deprecated=true]; - // The query params (if parseable by the membrane) - map query_params = 4; + // The old query params (preserving for backwards compatibility) + // TODO: Remove in 1.0 + map query_params_old = 4 [deprecated=true]; // HTTP request headers map headers = 5; + + // HTTP Query params + map query_params = 6; } message TopicTriggerContext { diff --git a/go.mod b/go.mod index b0c41ae8d..b44fa8329 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.6 + golang.org/x/tools v0.1.7 google.golang.org/api v0.40.0 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c google.golang.org/grpc v1.35.0 diff --git a/go.sum b/go.sum index dbad3bd2f..ac490cb3b 100644 --- a/go.sum +++ b/go.sum @@ -593,8 +593,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.6 h1:SIasE1FVIQOWz2GEAHFOmoW7xchJcqlucjSULTL0Ag4= -golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/triggers/http_request.go b/pkg/triggers/http_request.go index 5a8aa09f6..19bfcc03e 100644 --- a/pkg/triggers/http_request.go +++ b/pkg/triggers/http_request.go @@ -22,8 +22,6 @@ import ( // HttpRequest - Storage information that captures a HTTP Request type HttpRequest struct { - // The original Headers - // Header *fasthttp.RequestHeader Header map[string][]string // The original body stream Body []byte @@ -32,7 +30,7 @@ type HttpRequest struct { // The original path Path string // URL query parameters - Query map[string]string + Query map[string][]string } func (*HttpRequest) GetTriggerType() TriggerType { @@ -42,7 +40,7 @@ func (*HttpRequest) GetTriggerType() TriggerType { // FromHttpRequest (constructs a HttpRequest source type from a HttpRequest) func FromHttpRequest(ctx *fasthttp.RequestCtx) *HttpRequest { headerCopy := make(map[string][]string) - queryArgs := make(map[string]string) + queryArgs := make(map[string][]string) ctx.Request.Header.VisitAll(func(key []byte, val []byte) { keyString := string(key) @@ -60,7 +58,13 @@ func FromHttpRequest(ctx *fasthttp.RequestCtx) *HttpRequest { }) ctx.QueryArgs().VisitAll(func(key []byte, val []byte) { - queryArgs[string(key)] = string(val) + k := string(key) + + if queryArgs[k] == nil { + queryArgs[k] = make([]string, 0) + } + + queryArgs[k] = append(queryArgs[k], string(val)) }) return &HttpRequest{ diff --git a/pkg/worker/faas_http_worker.go b/pkg/worker/faas_http_worker.go index 322404ca5..7d06358ec 100644 --- a/pkg/worker/faas_http_worker.go +++ b/pkg/worker/faas_http_worker.go @@ -124,6 +124,13 @@ func (h *FaasHttpWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*trig } } + query := make(map[string]*pb.QueryValue) + for k, v := range trigger.Query { + query[k] = &pb.QueryValue{ + Value: v, + } + } + triggerRequest := &pb.TriggerRequest{ Data: trigger.Body, MimeType: mimeType, @@ -132,7 +139,7 @@ func (h *FaasHttpWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*trig Path: trigger.Path, Headers: headers, Method: trigger.Method, - QueryParams: trigger.Query, + QueryParams: query, }, }, } diff --git a/pkg/worker/faas_worker.go b/pkg/worker/faas_worker.go index b24c5d6f3..a21ac5fe0 100644 --- a/pkg/worker/faas_worker.go +++ b/pkg/worker/faas_worker.go @@ -62,7 +62,7 @@ func (s *FaasWorker) resolveTicket(ID string) (chan *pb.TriggerResponse, error) }() if s.responseQueue[ID] == nil { - return nil, fmt.Errorf("Attempted to resolve ticket that does not exist!") + return nil, fmt.Errorf("attempted to resolve ticket that does not exist!") } return s.responseQueue[ID], nil @@ -94,17 +94,30 @@ func (s *FaasWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*triggers } } + query := make(map[string]*pb.QueryValue) + queryOld := make(map[string]string) + for k, v := range trigger.Query { + if v != nil { + query[k] = &pb.QueryValue{ + Value: v, + } + if len(v) > 0 { + queryOld[k] = v[0] + } + } + } + triggerRequest := &pb.TriggerRequest{ Data: trigger.Body, MimeType: mimeType, Context: &pb.TriggerRequest_Http{ Http: &pb.HttpTriggerContext{ - Path: trigger.Path, - Method: trigger.Method, - QueryParams: trigger.Query, - Headers: headers, - HeadersOld: headersOld, - // TODO: Populate path params + Path: trigger.Path, + Method: trigger.Method, + QueryParams: query, + QueryParamsOld: queryOld, + Headers: headers, + HeadersOld: headersOld, }, }, } @@ -131,7 +144,7 @@ func (s *FaasWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*triggers httpResponse := triggerResponse.GetHttp() if httpResponse == nil { - return nil, fmt.Errorf("Fatal: Error handling event, incorrect response received from function") + return nil, fmt.Errorf("fatal: Error handling event, incorrect response received from function") } fasthttpHeader := &fasthttp.ResponseHeader{} diff --git a/pkg/worker/http_worker.go b/pkg/worker/http_worker.go index a97810b7a..acda63a53 100644 --- a/pkg/worker/http_worker.go +++ b/pkg/worker/http_worker.go @@ -64,7 +64,9 @@ func (h *HttpWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*triggers httpRequest.SetRequestURI(address) for key, val := range trigger.Query { - httpRequest.URI().QueryArgs().Add(key, val) + for _, v := range val { + httpRequest.URI().QueryArgs().Add(key, v) + } } for key, val := range trigger.Header { From 9f283687cfc00e81a59bbcada409fb134d1abff4 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 5 Oct 2021 12:18:24 +1100 Subject: [PATCH 70/83] fix: Update lambda API gateway and tests to support multi query parameters. --- pkg/plugins/gateway/lambda/lambda.go | 22 ++++++++++++++-------- pkg/plugins/gateway/lambda/lambda_test.go | 9 +++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pkg/plugins/gateway/lambda/lambda.go b/pkg/plugins/gateway/lambda/lambda.go index 9c71143b9..d3cc751f4 100644 --- a/pkg/plugins/gateway/lambda/lambda.go +++ b/pkg/plugins/gateway/lambda/lambda.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/url" "strings" "github.com/nitric-dev/membrane/pkg/triggers" @@ -137,14 +138,19 @@ func (event *Event) UnmarshalJSON(data []byte) error { // Copy the cookies over headerCopy["Cookie"] = evt.Cookies - event.Requests = append(event.Requests, &triggers.HttpRequest{ - // FIXME: Translate to http.Header - Header: headerCopy, - Body: []byte(evt.Body), - Method: evt.RequestContext.HTTP.Method, - Path: evt.RawPath, - Query: evt.QueryStringParameters, - }) + // Parse the raw query string + qVals, err := url.ParseQuery(evt.RawQueryString) + + if err == nil { + event.Requests = append(event.Requests, &triggers.HttpRequest{ + // FIXME: Translate to http.Header + Header: headerCopy, + Body: []byte(evt.Body), + Method: evt.RequestContext.HTTP.Method, + Path: evt.RawPath, + Query: qVals, + }) + } } break diff --git a/pkg/plugins/gateway/lambda/lambda_test.go b/pkg/plugins/gateway/lambda/lambda_test.go index 581ba6295..422283909 100644 --- a/pkg/plugins/gateway/lambda/lambda_test.go +++ b/pkg/plugins/gateway/lambda/lambda_test.go @@ -81,8 +81,9 @@ var _ = Describe("Lambda", func() { "x-nitric-request-id": "test-request-id", "Content-Type": "text/plain", }, - RawPath: "/test/test", - Body: "Test Payload", + RawPath: "/test/test", + RawQueryString: "key=test&key2=test1&key=test2", + Body: "Test Payload", RequestContext: events.APIGatewayV2HTTPRequestContext{ HTTP: events.APIGatewayV2HTTPRequestContextHTTPDescription{ Method: "GET", @@ -117,6 +118,10 @@ var _ = Describe("Lambda", func() { Expect(request.Method).To(Equal("GET")) By("Retaining the path") Expect(request.Path).To(Equal("/test/test")) + + By("Retaining the query parameters") + Expect(request.Query["key"]).To(BeEquivalentTo([]string{"test", "test2"})) + Expect(request.Query["key2"]).To(BeEquivalentTo([]string{"test1"})) }) }) }) From 9f0b955f18966f726b71220437d1456fe7c459f7 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Tue, 5 Oct 2021 13:50:32 +1100 Subject: [PATCH 71/83] ci: Remove github actions references to GOLANG_TOKEN. --- .github/workflows/test.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 85a53f7c0..e2aa104ad 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,9 +15,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - with: - token: ${{secrets.GOLANG_TOKEN}} - submodules: recursive - name: Setup Go uses: actions/setup-go@v2 with: @@ -49,9 +46,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - with: - token: ${{secrets.GOLANG_TOKEN}} - submodules: recursive - name: Setup Go uses: actions/setup-go@v2 with: From e853439f08c3bdaa5706cda5b9fc149c352908fe Mon Sep 17 00:00:00 2001 From: Rak Date: Thu, 7 Oct 2021 18:56:46 -0600 Subject: [PATCH 72/83] chore: generate mocks for sqs --- makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/makefile b/makefile index e69da274e..831e8d2bf 100644 --- a/makefile +++ b/makefile @@ -197,6 +197,7 @@ generate-mocks: @mkdir -p mocks/secrets_manager @mkdir -p mocks/key_vault @mkdir -p mocks/s3 + @mkdir -p mocks/sqs @mkdir -p mocks/azblob @mkdir -p mocks/mock_event_grid @mkdir -p mocks/azqueue @@ -205,6 +206,7 @@ generate-mocks: @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/storage/azblob/iface AzblobServiceUrlIface,AzblobContainerUrlIface,AzblobBlockBlobUrlIface,AzblobDownloadResponse > mocks/azblob/mock.go @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/key_vault/mock.go @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/sqs/sqsiface SQSAPI > mocks/sqs/mock.go @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi BaseClientAPI > mocks/mock_event_grid/mock.go @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi TopicsClientAPI > mocks/mock_event_grid/topic.go @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/queue/azqueue/iface AzqueueServiceUrlIface,AzqueueQueueUrlIface,AzqueueMessageUrlIface,AzqueueMessageIdUrlIface,DequeueMessagesResponseIface > mocks/azqueue/mock.go \ No newline at end of file From 286e8ef550a5c55ddf61d98930dfe54d559facf8 Mon Sep 17 00:00:00 2001 From: Rak Date: Thu, 7 Oct 2021 18:57:33 -0600 Subject: [PATCH 73/83] chore: add tech debt note for queue plugin interface --- pkg/plugins/queue/plugin.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/plugins/queue/plugin.go b/pkg/plugins/queue/plugin.go index b54f058a6..f8fa8f96c 100644 --- a/pkg/plugins/queue/plugin.go +++ b/pkg/plugins/queue/plugin.go @@ -77,6 +77,7 @@ type UnimplementedQueuePlugin struct { // Ensure UnimplementedQueuePlugin conforms to QueueService interface var _ QueueService = (*UnimplementedQueuePlugin)(nil) +// TODO: replace NitricTask and []NitricTask with pointers // Push - Unimplemented Stub for the UnimplementedQueuePlugin func (*UnimplementedQueuePlugin) Send(queue string, task NitricTask) error { return fmt.Errorf("UNIMPLEMENTED") From 48450e61fd92597b5bc113e2ab352b6c1c4a2b83 Mon Sep 17 00:00:00 2001 From: Rak Date: Thu, 7 Oct 2021 18:58:15 -0600 Subject: [PATCH 74/83] chore: remove old sqs mocks --- tests/mocks/sqs/mocks.go | 141 --------------------------------------- 1 file changed, 141 deletions(-) delete mode 100644 tests/mocks/sqs/mocks.go diff --git a/tests/mocks/sqs/mocks.go b/tests/mocks/sqs/mocks.go deleted file mode 100644 index d0b14dd3e..000000000 --- a/tests/mocks/sqs/mocks.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2021 Nitric Pty Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mocks_sqs - -import ( - "fmt" - "time" - - "github.com/aws/aws-sdk-go/service/sqs" - "github.com/aws/aws-sdk-go/service/sqs/sqsiface" -) - -type MockSqsOptions struct { - Queues []string - Messages map[string][]*Message - CompleteError error -} - -type Message struct { - Id *string - ReceiptHandle *string - Body *string -} - -type MockSqs struct { - sqsiface.SQSAPI - queues []string - messages map[string][]*Message - completeError error -} - -func (s *MockSqs) ListQueues(in *sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error) { - queueUrls := make([]*string, 0) - - for _, queue := range s.queues { - queueUrls = append(queueUrls, &queue) - } - - return &sqs.ListQueuesOutput{ - QueueUrls: queueUrls, - }, nil -} - -func (s *MockSqs) DeleteMessage(req *sqs.DeleteMessageInput) (*sqs.DeleteMessageOutput, error) { - // If an error has been set on the mock, return it. - if s.completeError != nil { - return nil, s.completeError - } - return &sqs.DeleteMessageOutput{}, nil -} - -func (s *MockSqs) ReceiveMessage(in *sqs.ReceiveMessageInput) (*sqs.ReceiveMessageOutput, error) { - for _, q := range s.queues { - if *in.QueueUrl == q { - mockMessages := s.messages[q] - - if mockMessages == nil || len(mockMessages) < 1 { - return &sqs.ReceiveMessageOutput{}, nil - } - - var messages []*sqs.Message - - for i, m := range mockMessages { - // Only return up to the max number of messages requested. - if int64(i) >= *in.MaxNumberOfMessages { - break - } - messages = append(messages, &sqs.Message{ - Body: m.Body, - ReceiptHandle: m.ReceiptHandle, - }) - mockMessages[i] = nil - } - - res := &sqs.ReceiveMessageOutput{ - Messages: messages, - } - - return res, nil - } - } - - return nil, fmt.Errorf("queue not found") -} - -func (s *MockSqs) SendMessageBatch(in *sqs.SendMessageBatchInput) (*sqs.SendMessageBatchOutput, error) { - for _, q := range s.queues { - if *in.QueueUrl == q { - if s.messages[q] == nil { - s.messages[q] = make([]*Message, 0) - } - - successfulMessages := make([]*sqs.SendMessageBatchResultEntry, 0) - failedTasks := make([]*sqs.BatchResultErrorEntry, 0) - for i, e := range in.Entries { - mockReceiptHandle := fmt.Sprintf("%s%s", string(rune(i)), time.Now()) - - s.messages[q] = append(s.messages[q], &Message{ - Id: e.Id, - Body: e.MessageBody, - ReceiptHandle: &mockReceiptHandle, - }) - - successfulMessages = append(successfulMessages, &sqs.SendMessageBatchResultEntry{ - Id: e.Id, - }) - } - - // TODO: Add a configurable failure mechanism here... - return &sqs.SendMessageBatchOutput{ - Successful: successfulMessages, - Failed: failedTasks, - }, nil - } - } - - return nil, fmt.Errorf("Queue: %s does not exist", *in.QueueUrl) -} - -func NewMockSqs(opts *MockSqsOptions) *MockSqs { - if opts.Messages == nil { - opts.Messages = make(map[string][]*Message) - } - return &MockSqs{ - queues: opts.Queues, - messages: opts.Messages, - completeError: opts.CompleteError, - } -} From bc414bef50059971817beb8fa27df179fb903da1 Mon Sep 17 00:00:00 2001 From: Rak Date: Thu, 7 Oct 2021 19:00:53 -0600 Subject: [PATCH 75/83] fix(plugin/sqs): use x-nitric-name to resolve queues --- pkg/plugins/queue/sqs/sqs.go | 48 ++- pkg/plugins/queue/sqs/sqs_suite_test.go | 2 +- pkg/plugins/queue/sqs/sqs_test.go | 388 ++++++++++++++++++------ 3 files changed, 323 insertions(+), 115 deletions(-) diff --git a/pkg/plugins/queue/sqs/sqs.go b/pkg/plugins/queue/sqs/sqs.go index fb8940013..b51c51679 100644 --- a/pkg/plugins/queue/sqs/sqs.go +++ b/pkg/plugins/queue/sqs/sqs.go @@ -17,7 +17,6 @@ package sqs_service import ( "encoding/json" "fmt" - "strings" "github.com/nitric-dev/membrane/pkg/plugins/errors" "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" @@ -25,32 +24,55 @@ import ( "github.com/nitric-dev/membrane/pkg/utils" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" ) +const ( + // ErrCodeNoSuchTagSet - AWS API neglects to include a constant for this error code. + ErrCodeNoSuchTagSet = "NoSuchTagSet" +) + type SQSQueueService struct { queue.UnimplementedQueuePlugin client sqsiface.SQSAPI } // Get the URL for a given queue name -func (s *SQSQueueService) getUrlForQueueName(queueName string) (*string, error) { - // TODO: Need to be able to guarantee same accound deployment in this case - // In this case it would be preferred to use this method - // s.client.GetQueueUrl(&sqs.GetQueueUrlInput{}) - if out, err := s.client.ListQueues(&sqs.ListQueuesInput{}); err == nil { - for _, url := range out.QueueUrls { - if strings.HasSuffix(*url, queueName) { - return url, nil +func (s *SQSQueueService) getUrlForQueueName(queue string) (*string, error) { + out, err := s.client.ListQueues(&sqs.ListQueuesInput{}) + + if err != nil { + return nil, fmt.Errorf("Encountered an error retrieving the queue list: %v", err) + } + + for _, q := range out.QueueUrls { + // TODO: This could be rather slow, it's interesting that they don't return this in the list queues output + tagout, err := s.client.ListQueueTags(&sqs.ListQueueTagsInput{ + QueueUrl: q, + }) + + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + // Table not found, try to create and put again + if awsErr.Code() == ErrCodeNoSuchTagSet { + // Ignore queues with no tags, check the next queue + continue + } + return nil, err } + return nil, err } - } else { - return nil, fmt.Errorf("An Unexpected error occurred: %s", err) - } - return nil, fmt.Errorf("Could not find Queue: %s", queueName) + for k, v := range tagout.Tags { + if k == "x-nitric-name" && *v == queue { + return q, nil + } + } + } + return nil, fmt.Errorf("Unable to find queue with name: %s", queue) } func (s *SQSQueueService) Send(queueName string, task queue.NitricTask) error { diff --git a/pkg/plugins/queue/sqs/sqs_suite_test.go b/pkg/plugins/queue/sqs/sqs_suite_test.go index 58f598dbf..ab57c3874 100644 --- a/pkg/plugins/queue/sqs/sqs_suite_test.go +++ b/pkg/plugins/queue/sqs/sqs_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqs_service_test +package sqs_service import ( "testing" diff --git a/pkg/plugins/queue/sqs/sqs_test.go b/pkg/plugins/queue/sqs/sqs_test.go index a253c6eb2..e55ee469f 100644 --- a/pkg/plugins/queue/sqs/sqs_test.go +++ b/pkg/plugins/queue/sqs/sqs_test.go @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqs_service_test +package sqs_service import ( - "encoding/json" "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/events" - sqs_service "github.com/nitric-dev/membrane/pkg/plugins/queue/sqs" - mocks_sqs "github.com/nitric-dev/membrane/tests/mocks/sqs" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" + "github.com/golang/mock/gomock" + mocks_sqs "github.com/nitric-dev/membrane/mocks/sqs" "github.com/nitric-dev/membrane/pkg/plugins/queue" . "github.com/onsi/ginkgo" @@ -28,16 +28,112 @@ import ( ) var _ = Describe("Sqs", func() { - // Tests for the BatchPush method - Context("BatchPush", func() { - When("Publishing to a queue that exists", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"test"}, + Context("getUrlForQueueName", func() { + When("List queues returns an error", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock).(*SQSQueueService) + + By("Calling ListQueues and receiving an error") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(nil, fmt.Errorf("mock-error")) + + _, err := plugin.getUrlForQueueName("test-queue") + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Encountered an error retrieving the queue list: mock-error")) + ctrl.Finish() }) - plugin := sqs_service.NewWithClient(sqsMock) + }) + + When("No queues exist", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock).(*SQSQueueService) + + By("Calling ListQueues and receiving no queue") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{}, + }, nil) - It("Should publish the message", func() { - _, err := plugin.SendBatch("test", []queue.NitricTask{ + _, err := plugin.getUrlForQueueName("test-queue") + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Unable to find queue with name: test-queue")) + ctrl.Finish() + }) + }) + + When("No queue tags match the nitric name", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock).(*SQSQueueService) + + // Name is in the URL, but that's not important. + queueUrl := aws.String("https://example.com/test-queue") + + By("Calling ListQueues and receiving no queue") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags with the available queues") + sqsMock.EXPECT().ListQueueTags(&sqs.ListQueueTagsInput{QueueUrl: queueUrl}).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + // The nitric name tag doesn't match the expected queue name + "x-nitric-name": aws.String("not-test-queue"), + }, + }, nil) + + By("calling getUrlForQueueName with test-queue") + _, err := plugin.getUrlForQueueName("test-queue") + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Unable to find queue with name: test-queue")) + ctrl.Finish() + }) + }) + }) + + // Tests for the BatchPush method + Context("BatchSend", func() { + When("Sending to a queue that exists", func() { + It("Should send the task to the queue", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) + + queueUrl := aws.String("https://example.com/test-queue") + + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("test-queue"), + }, + }, nil) + + By("Calling SendMessageBatch with the expected batch entries") + sqsMock.EXPECT().SendMessageBatch(&sqs.SendMessageBatchInput{ + QueueUrl: queueUrl, + Entries: []*sqs.SendMessageBatchRequestEntry{ + { + Id: aws.String("1234"), + MessageBody: aws.String(`{"id":"1234","payloadType":"test-payload","payload":{"Test":"Test"}}`), + }, + }, + }).Return(&sqs.SendMessageBatchOutput{}, nil) + + _, err := plugin.SendBatch("test-queue", []queue.NitricTask{ { ID: "1234", PayloadType: "test-payload", @@ -47,139 +143,229 @@ var _ = Describe("Sqs", func() { }, }) + By("Not returning an error") Expect(err).ShouldNot(HaveOccurred()) + ctrl.Finish() }) + }) When("Publishing to a queue that doesn't exist", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{}) - plugin := sqs_service.NewWithClient(sqsMock) + When("List queues returns an error", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) - It("Should fail to publish the message", func() { - _, err := plugin.SendBatch("test", []queue.NitricTask{ - { - ID: "1234", - PayloadType: "test-payload", - Payload: map[string]interface{}{ - "Test": "Test", + By("Calling ListQueues and receiving an error") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(nil, fmt.Errorf("mock-error")) + + _, err := plugin.SendBatch("test-queue", []queue.NitricTask{ + { + ID: "1234", + PayloadType: "test-payload", + Payload: map[string]interface{}{ + "Test": "Test", + }, }, - }, - }) + }) - Expect(err).Should(HaveOccurred()) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Encountered an error retrieving the queue list: mock-error")) + ctrl.Finish() + }) }) }) }) - // Tests for the Pop method - Context("Pop", func() { - When("Popping from a queue that exists", func() { + // Tests for the Receive method + Context("Receive", func() { + When("Receive from a queue that exists", func() { When("There is a message on the queue", func() { - mockId := "mockmessageid" - mockReceiptHandle := "mockreceipthandle" - jsonBytes, _ := json.Marshal(events.NitricEvent{ - ID: "mockrequestid", - PayloadType: "mockpayloadtype", - Payload: map[string]interface{}{}, - }) - mockEventJson := string(jsonBytes) + It("Should receive the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) + + queueUrl := aws.String("https://example.com/test-queue") - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"mock-queue"}, - Messages: map[string][]*mocks_sqs.Message{ - "mock-queue": { + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("mock-queue"), + }, + }, nil) + + By("Calling ReceiveMessage with the expected inputs") + sqsMock.EXPECT().ReceiveMessage(&sqs.ReceiveMessageInput{ + MaxNumberOfMessages: aws.Int64(int64(10)), + MessageAttributeNames: []*string{ + aws.String(sqs.QueueAttributeNameAll), + }, + QueueUrl: queueUrl, + }).Times(1).Return(&sqs.ReceiveMessageOutput{ + Messages: []*sqs.Message{ { - Id: &mockId, - ReceiptHandle: &mockReceiptHandle, - Body: &mockEventJson, + ReceiptHandle: aws.String("mockreceipthandle"), + Body: aws.String(`{"id":"1234","payloadType":"test-payload","payload":{"Test":"Test"}}`), }, }, - }, - }) - plugin := sqs_service.NewWithClient(sqsMock) + }, nil) - depth := uint32(10) + depth := uint32(10) - It("Should pop the message", func() { - msg, err := plugin.Receive(queue.ReceiveOptions{ + By("Returning the task") + messages, err := plugin.Receive(queue.ReceiveOptions{ QueueName: "mock-queue", Depth: &depth, }) - Expect(msg).To(HaveLen(1)) - + Expect(messages).To(HaveLen(1)) + Expect(messages[0]).To(BeEquivalentTo(queue.NitricTask{ + ID: "1234", + PayloadType: "test-payload", + LeaseID: "mockreceipthandle", + Payload: map[string]interface{}{ + "Test": "Test", + }, + })) Expect(err).ShouldNot(HaveOccurred()) + + ctrl.Finish() }) }) + When("There are no messages on the queue", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"mock-queue"}, - Messages: map[string][]*mocks_sqs.Message{ - // Queue with empty message slice - "mock-queue": make([]*mocks_sqs.Message, 0), - }, - }) - plugin := sqs_service.NewWithClient(sqsMock) - depth := uint32(10) + It("Should receive no messages", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) - It("Should pop the message", func() { - msg, err := plugin.Receive(queue.ReceiveOptions{ + queueUrl := aws.String("https://example.com/test-queue") + + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("mock-queue"), + }, + }, nil) + + By("Calling ReceiveMessage with the expected inputs") + sqsMock.EXPECT().ReceiveMessage(&sqs.ReceiveMessageInput{ + MaxNumberOfMessages: aws.Int64(int64(10)), + MessageAttributeNames: []*string{ + aws.String(sqs.QueueAttributeNameAll), + }, + QueueUrl: queueUrl, + }).Times(1).Return(&sqs.ReceiveMessageOutput{ + Messages: []*sqs.Message{}, + }, nil) + + depth := uint32(10) + + msgs, err := plugin.Receive(queue.ReceiveOptions{ QueueName: "mock-queue", Depth: &depth, }) - Expect(len(msg)).To(Equal(0)) + By("Returning an empty array of tasks") + Expect(msgs).To(HaveLen(0)) + By("Not returning an error") Expect(err).ShouldNot(HaveOccurred()) + + ctrl.Finish() }) }) }) - //When("Popping from a queue that doesn't exist", func() { - When("Popping from a queue that doesn't exist", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{}, - }) - plugin := sqs_service.NewWithClient(sqsMock) + // Tests for the Complete method + Context("Complete", func() { + When("The message is successfully deleted from SQS", func() { - depth := uint32(10) + // No errors set on mock, 'complete' won't return an error. + It("Should successfully delete the task", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) - It("Should return an error", func() { - _, err := plugin.Receive(queue.ReceiveOptions{ - QueueName: "non-existent-queue", - Depth: &depth, - }) + queueUrl := aws.String("https://example.com/test-queue") - By("Returning an error") - Expect(err).Should(HaveOccurred()) - }) - }) - }) + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) - // Tests for the Complete method - Context("Complete", func() { - When("The message is successfully deleted from SQS", func() { - // No errors set on mock, 'complete' won't return an error. - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"test-queue"}, - }) - plugin := sqs_service.NewWithClient(sqsMock) + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("test-queue"), + }, + }, nil) - It("Should not return an error", func() { - err := plugin.Complete("test-queue", "test-id") - Expect(err).ShouldNot(HaveOccurred()) - }) - }) - When("The message fails to delete from SQS", func() { - // No errors set on mock, 'complete' won't return an error. - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - CompleteError: fmt.Errorf("mock complete error"), + By("Calling SQS with the queue url and task lease id") + sqsMock.EXPECT().DeleteMessage(&sqs.DeleteMessageInput{ + QueueUrl: queueUrl, + ReceiptHandle: aws.String("lease-id"), + }).Times(1).Return( + &sqs.DeleteMessageOutput{}, + nil, + ) + + err := plugin.Complete("test-queue", "lease-id") + + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + + ctrl.Finish() + }) }) - plugin := sqs_service.NewWithClient(sqsMock) - It("Should return an error", func() { - err := plugin.Complete("test-queue", "test-id") - Expect(err).Should(HaveOccurred()) + When("The message fails to delete from SQS", func() { + // No errors set on mock, 'complete' won't return an error. + It("Return an error", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) + + queueUrl := aws.String("http://example.com/queue") + + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("test-queue"), + }, + }, nil) + + By("Calling SQS with the queue url and task lease id") + sqsMock.EXPECT().DeleteMessage(&sqs.DeleteMessageInput{ + QueueUrl: queueUrl, + ReceiptHandle: aws.String("test-id"), + }).Return(nil, fmt.Errorf("mock-error")) + + err := plugin.Complete("test-queue", "test-id") + + By("returning the error") + Expect(err).Should(HaveOccurred()) + + ctrl.Finish() + }) }) }) }) From a4c8b244bd9be5d2e2f4d10761517d540b8c163b Mon Sep 17 00:00:00 2001 From: Rak Date: Thu, 7 Oct 2021 19:30:58 -0600 Subject: [PATCH 76/83] chore: tidy comments --- pkg/plugins/queue/sqs/sqs.go | 1 - pkg/plugins/storage/s3/s3.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/plugins/queue/sqs/sqs.go b/pkg/plugins/queue/sqs/sqs.go index b51c51679..46f3d5075 100644 --- a/pkg/plugins/queue/sqs/sqs.go +++ b/pkg/plugins/queue/sqs/sqs.go @@ -56,7 +56,6 @@ func (s *SQSQueueService) getUrlForQueueName(queue string) (*string, error) { if err != nil { if awsErr, ok := err.(awserr.Error); ok { - // Table not found, try to create and put again if awsErr.Code() == ErrCodeNoSuchTagSet { // Ignore queues with no tags, check the next queue continue diff --git a/pkg/plugins/storage/s3/s3.go b/pkg/plugins/storage/s3/s3.go index 7006af1cd..1aaeb53aa 100644 --- a/pkg/plugins/storage/s3/s3.go +++ b/pkg/plugins/storage/s3/s3.go @@ -60,7 +60,6 @@ func (s *S3StorageService) getBucketByName(bucket string) (*s3.Bucket, error) { if err != nil { if awsErr, ok := err.(awserr.Error); ok { - // Table not found, try to create and put again if awsErr.Code() == ErrCodeNoSuchTagSet { // Ignore buckets with no tags, check the next bucket continue From e29d31e0ce90e582448d637e94d0af473710f2d3 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Mon, 11 Oct 2021 11:33:22 +1100 Subject: [PATCH 77/83] chore: Fix codecov badge link. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f6410a35..db5a01023 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

![test status](https://github.com/nitrictech/go-sdk/actions/workflows/test.yaml/badge.svg?branch=main) -[![codecov](https://codecov.io/gh/nitrictech/membrane/branch/main/graph/badge.svg?token=20TYFIQS2P)](https://codecov.io/gh/nitrictech/membrane) +[![codecov](https://codecov.io/gh/nitrictech/nitric/branch/develop/graph/badge.svg?token=20TYFIQS2P)](https://codecov.io/gh/nitrictech/nitric) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=nitrictech_membrane&metric=alert_status)](https://sonarcloud.io/dashboard?id=nitrictech_membrane) Nitric is a portable, provider independent runtime for cloud-native and serverless applications. Using Nitric applications can take advantage of cloud-native services for activities like eventing, queues, compute, CDN, storage, caches, etc. without direct integration to product specific APIs. From 59c7eb98095fe227b806dd0e2adf2f15e4e3dc5a Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Wed, 6 Oct 2021 10:11:30 +1100 Subject: [PATCH 78/83] feat(storage): Add minio storage plugin --- pkg/plugins/storage/minio/minio.go | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 pkg/plugins/storage/minio/minio.go diff --git a/pkg/plugins/storage/minio/minio.go b/pkg/plugins/storage/minio/minio.go new file mode 100644 index 000000000..5a67d9c04 --- /dev/null +++ b/pkg/plugins/storage/minio/minio.go @@ -0,0 +1,78 @@ +package storage_service + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/nitric-dev/membrane/pkg/plugins/storage" + s3_service "github.com/nitric-dev/membrane/pkg/plugins/storage/s3" + "github.com/nitric-dev/membrane/pkg/utils" +) + +const ( + MINIO_ENDPOINT_ENV = "MINIO_ENDPOINT" + MINIO_ACCESS_KEY_ENV = "MINIO_ACCESS_KEY" + MINIO_SECRET_KEY_ENV = "MINIO_SECRET_KEY" +) + +type minioConfig struct { + endpoint string + accessKey string + secretKey string +} + +func configFromEnv() (*minioConfig, error) { + endpoint := utils.GetEnv(MINIO_ENDPOINT_ENV, "") + accKey := utils.GetEnv(MINIO_ACCESS_KEY_ENV, "") + secKey := utils.GetEnv(MINIO_SECRET_KEY_ENV, "") + + configErrors := make([]error, 0) + + if endpoint == "" { + configErrors = append(configErrors, fmt.Errorf("%s not configured", MINIO_ENDPOINT_ENV)) + } + + if accKey == "" { + configErrors = append(configErrors, fmt.Errorf("%s not configured", MINIO_ACCESS_KEY_ENV)) + } + + if secKey == "" { + configErrors = append(configErrors, fmt.Errorf("%s not configured", MINIO_SECRET_KEY_ENV)) + } + + if len(configErrors) > 0 { + return nil, fmt.Errorf("configuration errors: %v", configErrors) + } + + return &minioConfig{ + endpoint: endpoint, + accessKey: accKey, + secretKey: secKey, + }, nil +} + +func New() (storage.StorageService, error) { + + conf, err := configFromEnv() + + if err != nil { + return nil, err + } + + // Configure to use MinIO Server + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(conf.accessKey, conf.secretKey, ""), + Endpoint: aws.String(conf.endpoint), + Region: aws.String("us-east-1"), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + newSession := session.New(s3Config) + + s3Client := s3.New(newSession) + + return s3_service.NewWithClient(s3Client) +} From 0e4b75824a62331465d3bbdc5adb862ae60823a6 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Wed, 6 Oct 2021 12:55:09 +1100 Subject: [PATCH 79/83] chore: storage plugin minor fixes. --- go.sum | 2 ++ pkg/plugins/storage/minio/minio.go | 8 ++++++-- pkg/plugins/storage/s3/s3.go | 13 ++++++++++--- pkg/providers/dev/membrane.go | 4 ++-- pkg/providers/dev/plugin.go | 4 ++-- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/go.sum b/go.sum index ac490cb3b..ebe9c52a3 100644 --- a/go.sum +++ b/go.sum @@ -593,6 +593,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.6 h1:SIasE1FVIQOWz2GEAHFOmoW7xchJcqlucjSULTL0Ag4= +golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/plugins/storage/minio/minio.go b/pkg/plugins/storage/minio/minio.go index 5a67d9c04..540f3949c 100644 --- a/pkg/plugins/storage/minio/minio.go +++ b/pkg/plugins/storage/minio/minio.go @@ -1,4 +1,4 @@ -package storage_service +package minio_storage_service import ( "fmt" @@ -70,7 +70,11 @@ func New() (storage.StorageService, error) { DisableSSL: aws.Bool(true), S3ForcePathStyle: aws.Bool(true), } - newSession := session.New(s3Config) + newSession, err := session.NewSession(s3Config) + + if err != nil { + return nil, fmt.Errorf("Error creating new session") + } s3Client := s3.New(newSession) diff --git a/pkg/plugins/storage/s3/s3.go b/pkg/plugins/storage/s3/s3.go index 1aaeb53aa..ecdba5d62 100644 --- a/pkg/plugins/storage/s3/s3.go +++ b/pkg/plugins/storage/s3/s3.go @@ -49,10 +49,17 @@ func (s *S3StorageService) getBucketByName(bucket string) (*s3.Bucket, error) { out, err := s.client.ListBuckets(&s3.ListBucketsInput{}) if err != nil { - return nil, fmt.Errorf("Encountered an error retrieving the bucket list: %v", err) + return nil, fmt.Errorf("encountered an error retrieving the bucket list: %v", err) } for _, b := range out.Buckets { + // Exact match bucket names required for minio + // Need to see if we can somehow tag, but directory names + // follow bucket names at the moment + if *b.Name == bucket { + return b, nil + } + // TODO: This could be rather slow, it's interesting that they don't return this in the list buckets output tagout, err := s.client.GetBucketTagging(&s3.GetBucketTaggingInput{ Bucket: b.Name, @@ -76,7 +83,7 @@ func (s *S3StorageService) getBucketByName(bucket string) (*s3.Bucket, error) { } } - return nil, fmt.Errorf("Unable to find bucket with name: %s", bucket) + return nil, fmt.Errorf("unable to find bucket with name: %s", bucket) } // Read - Retrieves an item from a bucket @@ -249,7 +256,7 @@ func New() (storage.StorageService, error) { }) if sessionError != nil { - return nil, fmt.Errorf("Error creating new AWS session %v", sessionError) + return nil, fmt.Errorf("error creating new AWS session %v", sessionError) } s3Client := s3.New(sess) diff --git a/pkg/providers/dev/membrane.go b/pkg/providers/dev/membrane.go index 1bfc3c631..986d7106d 100644 --- a/pkg/providers/dev/membrane.go +++ b/pkg/providers/dev/membrane.go @@ -26,8 +26,8 @@ import ( events_service "github.com/nitric-dev/membrane/pkg/plugins/events/dev" gateway_plugin "github.com/nitric-dev/membrane/pkg/plugins/gateway/dev" queue_service "github.com/nitric-dev/membrane/pkg/plugins/queue/dev" - boltdb_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/boltdb" secret_service "github.com/nitric-dev/membrane/pkg/plugins/secret/dev" + minio_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/minio" ) func main() { @@ -41,7 +41,7 @@ func main() { eventsPlugin, _ := events_service.New() gatewayPlugin, _ := gateway_plugin.New() queuePlugin, _ := queue_service.New() - storagePlugin, _ := boltdb_storage_service.New() + storagePlugin, _ := minio_storage_service.New() m, err := membrane.New(&membrane.MembraneOptions{ DocumentPlugin: documentPlugin, diff --git a/pkg/providers/dev/plugin.go b/pkg/providers/dev/plugin.go index d9b461f14..cd1004b6c 100644 --- a/pkg/providers/dev/plugin.go +++ b/pkg/providers/dev/plugin.go @@ -24,7 +24,7 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/queue" queue_service "github.com/nitric-dev/membrane/pkg/plugins/queue/dev" "github.com/nitric-dev/membrane/pkg/plugins/storage" - boltdb_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/boltdb" + minio_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/minio" "github.com/nitric-dev/membrane/pkg/providers" ) @@ -57,5 +57,5 @@ func (p *DevServiceFactory) NewQueueService() (queue.QueueService, error) { // NewStorageService - Returns local dev storage plugin func (p *DevServiceFactory) NewStorageService() (storage.StorageService, error) { - return boltdb_storage_service.New() + return minio_storage_service.New() } From 8debb903c368b53d1bdf7f49ef4ff617419e0f18 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Thu, 7 Oct 2021 08:41:38 +1100 Subject: [PATCH 80/83] chore: Add missing copyright headers --- pkg/plugins/storage/minio/minio.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/plugins/storage/minio/minio.go b/pkg/plugins/storage/minio/minio.go index 540f3949c..46e6a96e4 100644 --- a/pkg/plugins/storage/minio/minio.go +++ b/pkg/plugins/storage/minio/minio.go @@ -1,3 +1,17 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package minio_storage_service import ( From f48e01fbdfe447926a409851b72c3b0ff3544d5f Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Fri, 8 Oct 2021 13:39:56 +1100 Subject: [PATCH 81/83] chore: Add selectors to separate out bucket selection logic. --- pkg/plugins/storage/minio/minio.go | 12 ++++- pkg/plugins/storage/s3/option.go | 33 +++++++++++++ pkg/plugins/storage/s3/s3.go | 77 ++++++++++++++++++++---------- 3 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 pkg/plugins/storage/s3/option.go diff --git a/pkg/plugins/storage/minio/minio.go b/pkg/plugins/storage/minio/minio.go index 46e6a96e4..af34096f0 100644 --- a/pkg/plugins/storage/minio/minio.go +++ b/pkg/plugins/storage/minio/minio.go @@ -68,6 +68,14 @@ func configFromEnv() (*minioConfig, error) { }, nil } +func nameSelector(nitricName string, bucket *s3.Bucket) (bool, error) { + if *bucket.Name == nitricName { + return true, nil + } + + return false, nil +} + func New() (storage.StorageService, error) { conf, err := configFromEnv() @@ -87,10 +95,10 @@ func New() (storage.StorageService, error) { newSession, err := session.NewSession(s3Config) if err != nil { - return nil, fmt.Errorf("Error creating new session") + return nil, fmt.Errorf("error creating new session") } s3Client := s3.New(newSession) - return s3_service.NewWithClient(s3Client) + return s3_service.NewWithClient(s3Client, s3_service.WithSelector(nameSelector)) } diff --git a/pkg/plugins/storage/s3/option.go b/pkg/plugins/storage/s3/option.go new file mode 100644 index 000000000..5054ecbed --- /dev/null +++ b/pkg/plugins/storage/s3/option.go @@ -0,0 +1,33 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3_service + +type S3StorageServiceOption interface { + Apply(*S3StorageService) +} + +type withSelector struct { + selector BucketSelector +} + +func (w *withSelector) Apply(service *S3StorageService) { + service.selector = w.selector +} + +func WithSelector(selector BucketSelector) S3StorageServiceOption { + return &withSelector{ + selector: selector, + } +} diff --git a/pkg/plugins/storage/s3/s3.go b/pkg/plugins/storage/s3/s3.go index ecdba5d62..7d13c87e3 100644 --- a/pkg/plugins/storage/s3/s3.go +++ b/pkg/plugins/storage/s3/s3.go @@ -41,7 +41,36 @@ const ( // S3StorageService - Is the concrete implementation of AWS S3 for the Nitric Storage Plugin type S3StorageService struct { //storage.UnimplementedStoragePlugin - client s3iface.S3API + client s3iface.S3API + selector BucketSelector +} + +type BucketSelector = func(nitricName string, b *s3.Bucket) (bool, error) + +func (s *S3StorageService) tagSelector(name string, bucket *s3.Bucket) (bool, error) { + // TODO: This could be rather slow, it's interesting that they don't return this in the list buckets output + tagout, err := s.client.GetBucketTagging(&s3.GetBucketTaggingInput{ + Bucket: bucket.Name, + }) + + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + // Table not found, try to create and put again + if awsErr.Code() == ErrCodeNoSuchTagSet { + // Ignore buckets with no tags, check the next bucket + return false, nil + } + } + return false, err + } + + for _, tag := range tagout.TagSet { + if *tag.Key == "x-nitric-name" && *tag.Value == name { + return true, nil + } + } + + return false, nil } // getBucketByName - Finds and returns a bucket by it's Nitric name @@ -53,33 +82,23 @@ func (s *S3StorageService) getBucketByName(bucket string) (*s3.Bucket, error) { } for _, b := range out.Buckets { - // Exact match bucket names required for minio - // Need to see if we can somehow tag, but directory names - // follow bucket names at the moment - if *b.Name == bucket { - return b, nil + var selected bool = false + var selectErr error = nil + + if s.selector == nil { + // if selector is undefined us the default selector + selected, selectErr = s.tagSelector(bucket, b) + } else { + // Use provided selector if one available + selected, selectErr = s.selector(bucket, b) } - // TODO: This could be rather slow, it's interesting that they don't return this in the list buckets output - tagout, err := s.client.GetBucketTagging(&s3.GetBucketTaggingInput{ - Bucket: b.Name, - }) - - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - if awsErr.Code() == ErrCodeNoSuchTagSet { - // Ignore buckets with no tags, check the next bucket - continue - } - return nil, err - } + if selectErr != nil { return nil, err } - for _, tag := range tagout.TagSet { - if *tag.Key == "x-nitric-name" && *tag.Value == bucket { - return b, nil - } + if selected { + return b, nil } } @@ -267,8 +286,14 @@ func New() (storage.StorageService, error) { } // NewWithClient creates a new S3 Storage plugin and injects the given client -func NewWithClient(client s3iface.S3API) (storage.StorageService, error) { - return &S3StorageService{ +func NewWithClient(client s3iface.S3API, opts ...S3StorageServiceOption) (storage.StorageService, error) { + s3Client := &S3StorageService{ client: client, - }, nil + } + + for _, o := range opts { + o.Apply(s3Client) + } + + return s3Client, nil } From ef28b44dd83a332806a60d05e9fc84d09040ded8 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Mon, 11 Oct 2021 15:34:14 +1100 Subject: [PATCH 82/83] chore: Remove unused mock struct. --- pkg/plugins/storage/azblob/azblob_test.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/pkg/plugins/storage/azblob/azblob_test.go b/pkg/plugins/storage/azblob/azblob_test.go index 4486e9928..5ce82ae8c 100644 --- a/pkg/plugins/storage/azblob/azblob_test.go +++ b/pkg/plugins/storage/azblob/azblob_test.go @@ -31,28 +31,6 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/storage" ) -type mockStorageCredential struct { - azblob.StorageAccountCredential -} - -//AccountName() string -// ComputeHMACSHA256(message string) (base64String string) -// getUDKParams() *UserDelegationKey - -func (m *mockStorageCredential) AccountName() string { - return "mock-account-name" -} - -func (m *mockStorageCredential) ComputeHMACSHA256(message string) (base64String string) { - base64String = "mock-string" - - return -} - -func (m *mockStorageCredential) getUDKParams() *azblob.UserDelegationKey { - return &azblob.UserDelegationKey{} -} - var _ = Describe("Azblob", func() { //Context("New", func() { // When("", func() { From 39bc5e242239d0c11b7c298741a24c46ad1e4425 Mon Sep 17 00:00:00 2001 From: Tim Holm Date: Thu, 14 Oct 2021 18:54:54 +1100 Subject: [PATCH 83/83] fix(queues/pubsub): Properly return AckId. --- pkg/plugins/queue/pubsub/pubsub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugins/queue/pubsub/pubsub.go b/pkg/plugins/queue/pubsub/pubsub.go index 65f56d684..e58b61c04 100644 --- a/pkg/plugins/queue/pubsub/pubsub.go +++ b/pkg/plugins/queue/pubsub/pubsub.go @@ -256,7 +256,7 @@ func (s *PubsubQueueService) Receive(options queue.ReceiveOptions) ([]queue.Nitr ID: nitricTask.ID, Payload: nitricTask.Payload, PayloadType: nitricTask.PayloadType, - LeaseID: nitricTask.LeaseID, + LeaseID: m.AckId, }) }