diff --git a/.github/workflows/generate-sdks.yaml b/.github/workflows/generate-sdks.yaml new file mode 100644 index 000000000..bd925181c --- /dev/null +++ b/.github/workflows/generate-sdks.yaml @@ -0,0 +1,23 @@ +name: Generate SDKs +on: + release: + types: [published] + +jobs: + # Bump the membrane version + generate_sdks: + name: Signal base SDK repo for auto regen + runs-on: ubuntu-latest + steps: + - name: Signal base SDK repo + uses: peter-evans/repository-dispatch@v1 + with: + token: ${{ secrets.NITRIC_BOT_TOKEN }} + repository: nitrictech/apis + event-type: generate + client-payload: > + { + "prerelease": ${{ github.event.release.prerelease }}, + "tag_name": "${{ github.event.release.tag_name }}" + } + \ No newline at end of file diff --git a/.github/workflows/publish-binaries.yaml b/.github/workflows/publish-binaries.yaml new file mode 100644 index 000000000..190660011 --- /dev/null +++ b/.github/workflows/publish-binaries.yaml @@ -0,0 +1,71 @@ +name: Release Binaries +on: + release: + types: [published] + +jobs: + add_binaries: + name: Add Binaries to release + runs-on: ubuntu-latest + env: + GOPATH: /home/runner/go + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.7 + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: Install modules + run: make install-tools + - name: Make binaries + run: make build-all-binaries + - name: Upload Dev + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-dev + asset_name: membrane-dev + asset_content_type: application/octet-stream + - name: Upload AWS + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-aws + asset_name: membrane-aws + asset_content_type: application/octet-stream + - name: Upload GCP + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-gcp + asset_name: membrane-gcp + asset_content_type: application/octet-stream + - name: Upload Azure + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-azure + asset_name: membrane-azure + asset_content_type: application/octet-stream + - name: Upload Digital Ocean + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./bin/membrane-do + asset_name: membrane-do + asset_content_type: application/octet-stream \ No newline at end of file diff --git a/.github/workflows/publish-contracts.yaml b/.github/workflows/publish-contracts.yaml new file mode 100644 index 000000000..9f8d2b60b --- /dev/null +++ b/.github/workflows/publish-contracts.yaml @@ -0,0 +1,41 @@ +name: Release Contracts +on: + release: + types: [published] + +jobs: + # Bump the membrane version + contract_release: + name: Add contracts to release + runs-on: ubuntu-latest + env: + GOPATH: /home/runner/go + steps: + - name: Checkout + uses: actions/checkout@v2 + + # Add validate contract to the released contracts + - name: Download validate contract + run: | + mkdir -p ./contracts/validate + curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.6.1/validate/validate.proto --output ./contracts/validate/validate.proto + + # Tarball the contracts repository + - name: Archive Release + uses: thedoctor0/zip-release@master + with: + type: 'tar' + filename: 'contracts.tgz' + path: contracts + + # Upload contract tarball to the releases + - name: Upload Contracts + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./contracts.tgz + asset_name: contracts.tgz + asset_content_type: application/tar+gzip + \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/publish-docker.yaml similarity index 61% rename from .github/workflows/release.yaml rename to .github/workflows/publish-docker.yaml index e3984f989..aa22432a4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/publish-docker.yaml @@ -1,7 +1,7 @@ name: Release Docker images on: release: - types: [released] + types: [published] jobs: push_to_registry: name: Push Docker image to Docker Hub @@ -9,12 +9,6 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v2 - with: - token: ${{secrets.GOLANG_TOKEN}} - submodules: recursive - - name: Get Tag - id: tag - run: echo ::set-output name=TAG::$(echo $GITHUB_REF | cut -d / -f 3) # Setup docker build image - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -24,6 +18,19 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Sets the latest tag to rc-latest if release was a prerelease + - name: Set latest tag RC + if: "github.event.release.prerelease" + run: | + echo "latest_tag=rc-latest" >> $GITHUB_ENV + + # Uses latest tag if release was a production release + - name: Set latest tag Prod + if: "!github.event.release.prerelease" + run: | + echo "latest_tag=latest" >> $GITHUB_ENV + # Push development image - name: Push Local Static image to Docker Hub uses: docker/build-push-action@v2 @@ -32,8 +39,8 @@ jobs: file: ./pkg/providers/dev/dev.dockerfile push: true tags: | - nitricimages/membrane-local:latest - nitricimages/membrane-local:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-local:${{ env.latest_tag }} + nitricimages/membrane-local:${{ github.event.release.tag_name }} # Push AWS image - name: Push AWS Static image to Docker Hub uses: docker/build-push-action@v2 @@ -42,8 +49,8 @@ jobs: file: ./pkg/providers/aws/aws.dockerfile push: true tags: | - nitricimages/membrane-aws:latest - nitricimages/membrane-aws:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-aws:${{ env.latest_tag }} + nitricimages/membrane-aws:${{ github.event.release.tag_name }} # Push GCP image - name: Push GCP Static image to Docker Hub uses: docker/build-push-action@v2 @@ -52,8 +59,8 @@ jobs: file: ./pkg/providers/gcp/gcp.dockerfile push: true tags: | - nitricimages/membrane-gcp:latest - nitricimages/membrane-gcp:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-gcp:${{ env.latest_tag }} + nitricimages/membrane-gcp:${{ github.event.release.tag_name }} # Push Azure image - name: Push Azure Static image to Docker Hub uses: docker/build-push-action@v2 @@ -62,8 +69,8 @@ jobs: file: ./pkg/providers/azure/azure.dockerfile push: true tags: | - nitricimages/membrane-azure:latest - nitricimages/membrane-azure:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-azure:${{ env.latest_tag }} + nitricimages/membrane-azure:${{ github.event.release.tag_name }} # Push Digital Ocean image - name: Push Digital Ocean image to Docker Hub uses: docker/build-push-action@v2 @@ -72,5 +79,5 @@ jobs: file: ./pkg/providers/do/do.dockerfile push: true tags: | - nitricimages/membrane-do:latest - nitricimages/membrane-do:${{ steps.tag.outputs.TAG }} + nitricimages/membrane-do:${{ env.latest_tag }} + nitricimages/membrane-do:${{ github.event.release.tag_name }} diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 53fd28521..000000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: Create Release -on: - pull_request: - types: [closed] - branches: - - 'main' -jobs: - # Bump the membrane version - version_bump: - if: github.event.pull_request.merged == true - name: Bump Version and Create Release - runs-on: ubuntu-latest - outputs: - version_id: ${{ steps.tag_version.outputs.new_tag }} - upload_url: ${{ steps.create_release.outputs.upload_url }} - steps: - - uses: actions/checkout@v2 - - name: Bump version and push tag - id: tag_version - uses: mathieudutour/github-tag-action@v5.5 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Create a GitHub release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: true - tag_name: ${{ steps.tag_version.outputs.new_tag }} - release_name: Release ${{ steps.tag_version.outputs.new_tag }} - body: ${{ steps.tag_version.outputs.changelog }} - - # Add membrane binaries to the release - add_binaries: - needs: version_bump - name: Add Binaries to release - runs-on: ubuntu-latest - env: - GOPATH: /home/runner/go - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Setup Go - uses: actions/setup-go@v2 - with: - go-version: 1.16.7 - - name: Install Protoc - uses: arduino/setup-protoc@v1 - - name: Install modules - run: make install-tools - - name: Make binaries - run: make build-all-binaries - - name: Upload Dev - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-dev - asset_name: membrane-dev - asset_content_type: application/octet-stream - - name: Upload AWS - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-aws - asset_name: membrane-aws - asset_content_type: application/octet-stream - - name: Upload GCP - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-gcp - asset_name: membrane-gcp - asset_content_type: application/octet-stream - - name: Upload Digital Ocean - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./bin/membrane-do - asset_name: membrane-do - asset_content_type: application/octet-stream - - # Add contracts to release - add_contracts: - needs: version_bump - name: Add contracts to release - runs-on: ubuntu-latest - env: - GOPATH: /home/runner/go - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Download validate contract - run: | - mkdir -p ./contracts/validate - curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.6.1/validate/validate.proto --output ./contracts/validate/validate.proto - - name: Archive Release - uses: thedoctor0/zip-release@master - with: - type: 'tar' - filename: 'contracts.tgz' - path: ./contracts - - name: Upload Contracts - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} - with: - upload_url: ${{ needs.version_bump.outputs.upload_url }} - asset_path: ./contracts.tgz - asset_name: contracts.tgz - asset_content_type: application/tar+gzip \ No newline at end of file diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml new file mode 100644 index 000000000..e6f88dee8 --- /dev/null +++ b/.github/workflows/release-prod.yaml @@ -0,0 +1,35 @@ +name: Production Release +on: + pull_request: + types: [closed] + branches: + - 'main' +jobs: + # Bump the membrane version + version_bump: + if: github.event.pull_request.merged == true + name: Bump Version and Create Release + runs-on: ubuntu-latest + outputs: + version_id: ${{ steps.tag_version.outputs.new_tag }} + upload_url: ${{ steps.create_release.outputs.upload_url }} + steps: + - uses: actions/checkout@v2 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + # Use GITHUB_TOKEN here to prevent further workflows + # generated on 'tag' action + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Create a GitHub release + id: create_release + uses: actions/create-release@v1 + env: + # Use NITRIC_BOT_TOKEN here to + # trigger release 'published' workflows + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + tag_name: ${{ steps.tag_version.outputs.new_tag }} + release_name: Release ${{ steps.tag_version.outputs.new_tag }} + body: ${{ steps.tag_version.outputs.changelog }} \ No newline at end of file diff --git a/.github/workflows/release-rc.yaml b/.github/workflows/release-rc.yaml new file mode 100644 index 000000000..533dfe261 --- /dev/null +++ b/.github/workflows/release-rc.yaml @@ -0,0 +1,45 @@ +name: Release Candidate +on: + pull_request: + types: [closed] + branches: + - 'develop' +jobs: + # Bump the membrane version + version_bump: + if: github.event.pull_request.merged == true + name: Bump Version and Create Release + runs-on: ubuntu-latest + outputs: + version_id: ${{ steps.tag_version.outputs.new_tag }} + upload_url: ${{ steps.create_release.outputs.upload_url }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + # Don't commit tag + # this will be done as part of the release + dry_run: true + github_token: ${{ secrets.GITHUB_TOKEN }} + release_branches: main,develop + + - name: Calculate RC number + id: vars + run: echo "::set-output name=rc_num::$(git rev-list --merges --count origin/develop...origin/main)" + + - name: Create a GitHub release + id: create_release + uses: actions/create-release@v1 + env: + # Use NITRIC_BOT_TOKEN here to + # trigger release 'published' workflows + GITHUB_TOKEN: ${{ secrets.NITRIC_BOT_TOKEN }} + with: + prerelease: true + tag_name: ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.rc_num }} + release_name: Release ${{ steps.tag_version.outputs.new_tag }}-rc.${{ steps.vars.outputs.rc_num }} + body: ${{ steps.tag_version.outputs.changelog }}-rc.${{ steps.vars.outputs.rc_num }} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4d00cc462..e2aa104ad 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,9 +4,9 @@ on: push: branches: - main + - develop pull_request: - jobs: test: runs-on: ubuntu-latest @@ -15,9 +15,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - with: - token: ${{secrets.GOLANG_TOKEN}} - submodules: recursive - name: Setup Go uses: actions/setup-go@v2 with: @@ -49,27 +46,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - with: - token: ${{secrets.GOLANG_TOKEN}} - submodules: recursive - name: Setup Go uses: actions/setup-go@v2 with: go-version: 1.16.7 - name: Install Protoc uses: arduino/setup-protoc@v1 - - name: Install modules - run: make install-tools - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - java-version: '11' - distribution: 'adopt' - - name: Setup local DynamoDB - run: make install-test-tools - - name: Setup Cloud SDK - uses: google-github-actions/setup-gcloud@master - - name: Install Google Cloud SDK components - run: yes | gcloud components install beta cloud-firestore-emulator - name: Run Integration Tests run: make test-integration \ No newline at end of file diff --git a/README.md b/README.md index c5176ee63..db5a01023 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

![test status](https://github.com/nitrictech/go-sdk/actions/workflows/test.yaml/badge.svg?branch=main) -[![codecov](https://codecov.io/gh/nitrictech/membrane/branch/main/graph/badge.svg?token=20TYFIQS2P)](https://codecov.io/gh/nitrictech/membrane) +[![codecov](https://codecov.io/gh/nitrictech/nitric/branch/develop/graph/badge.svg?token=20TYFIQS2P)](https://codecov.io/gh/nitrictech/nitric) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=nitrictech_membrane&metric=alert_status)](https://sonarcloud.io/dashboard?id=nitrictech_membrane) Nitric is a portable, provider independent runtime for cloud-native and serverless applications. Using Nitric applications can take advantage of cloud-native services for activities like eventing, queues, compute, CDN, storage, caches, etc. without direct integration to product specific APIs. @@ -24,10 +24,9 @@ The Membrane is at the heart of the solution. Nitric applications communicate wi Additional services on our roadmap include: - - RDS - - Configuration - - Logging - Authentication + - Email + - Logging > If you have any requests or suggestions let us know in the issues. @@ -35,11 +34,9 @@ SDKs are available for many languages, providing an idiomatic wrapper around the - [Node.js](https://github.com/nitrictech/node-sdk) - [Python](https://github.com/nitrictech/python-sdk) - - [Go](https://github.com/nitrictech/go-sdk) - [Java](https://github.com/nitrictech/java-sdk) - - [PHP](https://github.com/nitrictech/php-sdk) - - [.NET](https://github.com/nitrictech/dotnet-sdk) - + - [Go](https://github.com/nitrictech/go-sdk) + > If you have additional languages you would like support for let us know in the issues, we also welcome community contribtions for new language support. ## Architecture @@ -79,12 +76,10 @@ provider as an alternative to the fixed set of plugins in the static membranes. ### Requirements - Git - - Golang + - Golang (1.16) - Make - Docker - Google Protocol Buffers Compiler - - Google Cloud SDK (for Firestore testing) - - JRE (for DynamoDB testing) ### Getting Started @@ -93,29 +88,14 @@ provider as an alternative to the fixed set of plugins in the static membranes. make install-tools ``` -#### Install integration testing tools -```bash -make install-test-tools -``` - ##### Install Protocol Buffers Download the Google Protobuf Compiler (standalone binary called `protoc`) from https://github.com/protocolbuffers/protobuf and add it to your $PATH. > On MacOS with Homebrew, you can run `brew install protobuf` -##### Install Google Cloud SDK -Install the Google Cloud SDK following in the instructions at: https://cloud.google.com/sdk/docs/install - -#### Install JRE -Install a Java Runtime Environment (JRE) version 11 or later for your OS. For example on Ubuntu Linux run: - -```bash -sudo apt-get install openjdk-11-jdk -``` - ### Run unit tests ```bash -make tests +make test ``` ### Run integration tests ```bash diff --git a/contracts/proto/error/v1/error.proto b/contracts/proto/error/v1/error.proto new file mode 100644 index 000000000..485abe431 --- /dev/null +++ b/contracts/proto/error/v1/error.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; +package nitric.error.v1; + +// protoc plugin options for code generation +option go_package = "nitric/v1;v1"; +option java_package = "io.nitric.proto.error.v1"; +option java_multiple_files = true; +option java_outer_classname = "Errors"; +option php_namespace = "Nitric\\Proto\\Error\\V1"; +option csharp_namespace = "Nitric.Proto.Error.v1"; + +message ErrorScope { + // The API service invoked, e.g. 'Service.Method'. + string service = 1; + + // The plugin method invoked, e.g. 'PluginService.Method'. + string plugin = 2; + + // The plugin method arguments, ensure only non-sensitive data is specified. + map args = 3; +} + +message ErrorDetails { + // The developer error message, explaining the error and ideally solution. + string message = 1; + + // The error root cause. + string cause = 2; + + // The scope of the error. + ErrorScope scope = 3; +} \ No newline at end of file diff --git a/contracts/proto/faas/v1/faas.proto b/contracts/proto/faas/v1/faas.proto index ed5d69927..2bb4fc4e9 100644 --- a/contracts/proto/faas/v1/faas.proto +++ b/contracts/proto/faas/v1/faas.proto @@ -74,6 +74,10 @@ message HeaderValue { repeated string value = 1; } +message QueryValue { + repeated string value = 1; +} + message HttpTriggerContext { // The request method @@ -86,11 +90,15 @@ message HttpTriggerContext { // TODO: Remove in 1.0 map headers_old = 3 [deprecated=true]; - // The query params (if parseable by the membrane) - map query_params = 4; + // The old query params (preserving for backwards compatibility) + // TODO: Remove in 1.0 + map query_params_old = 4 [deprecated=true]; // HTTP request headers map headers = 5; + + // HTTP Query params + map query_params = 6; } message TopicTriggerContext { diff --git a/contracts/proto/storage/v1/storage.proto b/contracts/proto/storage/v1/storage.proto index 2be8aeeed..099205466 100644 --- a/contracts/proto/storage/v1/storage.proto +++ b/contracts/proto/storage/v1/storage.proto @@ -19,6 +19,8 @@ service StorageService { rpc Write (StorageWriteRequest) returns (StorageWriteResponse); // Delete an item from a bucket rpc Delete (StorageDeleteRequest) returns (StorageDeleteResponse); + // Generate a pre-signed URL for direct operations on an item + rpc PreSignUrl (StoragePreSignUrlRequest) returns (StoragePreSignUrlResponse); } // Request to put (create/update) a storage item @@ -68,4 +70,29 @@ message StorageDeleteRequest { } // Result of deleting a storage item -message StorageDeleteResponse {} \ No newline at end of file +message StorageDeleteResponse {} + +// Request to generate a pre-signed URL for a file to perform a specific operation, such as read or write. +message StoragePreSignUrlRequest { + // Nitric name of the bucket to retrieve from + // this will be automatically resolved to the provider specific bucket identifier. + string bucket_name = 1; + // Key of item to generate the signed URL for. + // The URL and the token it contains will only be valid for operations on this resource specifically. + string key = 2; + // Operation + enum Operation { + READ = 0; + WRITE = 1; + } + Operation operation = 3; + // Expiry time in seconds for the token included in the signed URL. + // Time starts from when the access token is generated, not when this request is made. + // e.g. time.Now().Add(expiry * time.Second) on the server + uint32 expiry = 4; +} + +message StoragePreSignUrlResponse { + // The pre-signed url, restricted to the operation, resource and expiry time specified in the request. + string url = 1; +} \ No newline at end of file diff --git a/go.mod b/go.mod index 5994a9cda..8b5a68509 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,15 @@ require ( cloud.google.com/go/firestore v1.5.0 cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.10.0 - github.com/Azure/azure-sdk-for-go v51.3.0+incompatible - github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/azure-sdk-for-go v56.3.0+incompatible + github.com/Azure/azure-storage-blob-go v0.13.0 + github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd + github.com/Azure/go-autorest/autorest v0.11.18 + github.com/Azure/go-autorest/autorest/adal v0.9.14 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 + github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/DataDog/zstd v1.4.8 // indirect github.com/Knetic/govaluate v3.0.0+incompatible @@ -16,12 +23,12 @@ require ( github.com/asdine/storm v2.1.2+incompatible github.com/aws/aws-lambda-go v1.20.0 github.com/aws/aws-sdk-go v1.36.12 - github.com/envoyproxy/protoc-gen-validate v0.6.1 // indirect + github.com/envoyproxy/protoc-gen-validate v0.6.1 github.com/golang/mock v1.4.4 github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.3 // indirect github.com/google/addlicense v1.0.0 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.2.0 github.com/googleapis/gax-go/v2 v2.0.5 github.com/mitchellh/mapstructure v1.4.1 github.com/onsi/ginkgo v1.16.4 @@ -30,11 +37,12 @@ require ( github.com/valyala/fasthttp v1.23.0 github.com/vmihailenco/msgpack v3.3.3+incompatible // indirect go.etcd.io/bbolt v1.3.5 - go.mongodb.org/mongo-driver v1.7.1 // indirect + go.mongodb.org/mongo-driver v1.7.1 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 - golang.org/x/tools v0.1.5 + golang.org/x/text v0.3.7 // indirect + golang.org/x/tools v0.1.7 google.golang.org/api v0.40.0 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c google.golang.org/grpc v1.35.0 diff --git a/go.sum b/go.sum index 9661fc7fb..5265462b6 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,8 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.5.0 h1:4qNItsmc4GP6UOZPGemmHY4ZfPofVhcaKXsYw9wm9oA= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= @@ -40,29 +38,46 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v51.3.0+incompatible h1:Y3wR7C5Sj0nZG3VhkePF5hK7zNCS5yeImN/k2CWB+u8= -github.com/Azure/azure-sdk-for-go v51.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= +github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc= +github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs= +github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= +github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.14 h1:G8hexQdV5D4khOXrWG2YuLCFKhWYmWD8bHYaXN5ophk= +github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 h1:DOhB+nXkF7LN0JfBGB5YtCF6QLK8mLe4psaHF7ZQEKM= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.3/go.mod h1:yAQ2b6eP/CmLPnmLvxtT1ALIY3OR1oFcCqVBi8vHiTc= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -80,19 +95,13 @@ github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -100,13 +109,15 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad h1:EmNYJhPYy0pOFjCx2PrgtaBXmee0iUX9hLlxE1xHOJE= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= github.com/envoyproxy/protoc-gen-validate v0.6.1/go.mod h1:txg5va2Qkip90uYoSKH+nkAAmXrb2j3iq4FLwdrCbXQ= @@ -115,10 +126,8 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -148,7 +157,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -182,18 +190,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/addlicense v0.0.0-20210428195630-6d92264d7170 h1:jLUa4MO3autxlRJmC4KubeE5QGIb5JqW9oEaqYTb/fA= -github.com/google/addlicense v0.0.0-20210428195630-6d92264d7170/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= -github.com/google/addlicense v0.0.0-20210727174409-874627749a46 h1:1locMH9PVZH3LXvogcvdTxf2/9J4YT/9W3BSXrTN4/U= -github.com/google/addlicense v0.0.0-20210727174409-874627749a46/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= -github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b h1:KwI0NOpYd3rzKojfjeRerF7rzjeTwvJARVsgGf5TWmY= -github.com/google/addlicense v0.0.0-20210729153508-ef04bb38a16b/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= -github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab h1:+qfOxKbnAqDNCoFUNHxudKs8Z14T5EBYntAeWIeI1eA= -github.com/google/addlicense v0.0.0-20210810170408-9cc7ec3e36ab/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/addlicense v1.0.0 h1:cqvo5suPWlsk6r6o42Fs2K66xYCl2tnhVPUYoP3EnO4= github.com/google/addlicense v1.0.0/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -222,12 +221,12 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2 h1:LR89qFljJ48s990kEKGsk213yIJDPI4205OKOzbURK8= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -236,14 +235,11 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7 h1:ux/56T2xqZO/3cP1I2F86qpeoYPCOzk+KF/UH/Ar+lk= github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -256,18 +252,14 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -277,23 +269,27 @@ github.com/lyft/protoc-gen-star v0.5.1 h1:sImehRT+p7lW9n6R7MQc5hVgzWGEkDVZU4AsBQ github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/muesli/termenv v0.7.4 h1:/pBqvU5CpkY53tU0vVn+xgs2ZTX63aH5nY+SSps5Xa8= github.com/muesli/termenv v0.7.4/go.mod h1:pZ7qY9l3F7e5xsAOS0zCew2tME+p7bWeBkotCEcIIcc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= @@ -303,15 +299,12 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -327,7 +320,6 @@ github.com/spf13/afero v1.3.4 h1:8q6vk3hthlpb2SouZcnBVKboxWQWMDNF38bwholZrJc= github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -336,19 +328,17 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/uw-labs/lichen v0.1.4 h1:ZdTT7u0I3MI65CCDaNpMBh3VzhOf1Xbw0gF2VRKQ7/4= -github.com/uw-labs/lichen v0.1.4 h1:ZdTT7u0I3MI65CCDaNpMBh3VzhOf1Xbw0gF2VRKQ7/4= -github.com/uw-labs/lichen v0.1.4/go.mod h1:Fljba7ubiwHbEp1nzUjth+cWUbyQe73WUyL1++5HRNY= github.com/uw-labs/lichen v0.1.4/go.mod h1:Fljba7ubiwHbEp1nzUjth+cWUbyQe73WUyL1++5HRNY= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -364,8 +354,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA= @@ -387,8 +376,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -398,10 +388,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -417,7 +405,6 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -442,6 +429,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -463,8 +451,10 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -507,6 +497,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -524,6 +515,7 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -533,11 +525,10 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210223095934-7937bea0104d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -545,11 +536,12 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -604,8 +596,10 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6 h1:SIasE1FVIQOWz2GEAHFOmoW7xchJcqlucjSULTL0Ag4= +golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -709,11 +703,11 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -733,11 +727,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/makefile b/makefile index 42977531a..831e8d2bf 100644 --- a/makefile +++ b/makefile @@ -18,13 +18,6 @@ install-tools: install @echo Installing tools from tools.go @cat ./tools/tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go get % -# Install integration testing tools -install-test-tools: - @wget https://s3.us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.tar.gz - @sudo mkdir -p /usr/local/dynamodb - @sudo tar -xf dynamodb_local_latest.tar.gz -C /usr/local/dynamodb - @rm dynamodb_local_latest.tar.gz - clean: @rm -rf ./bin/ @rm -rf ./lib/ @@ -183,6 +176,7 @@ build-all-binaries: clean generate-proto @echo Building all provider membranes @CGO_ENABLED=0 go build -o bin/membrane-gcp -ldflags="-extldflags=-static" ./pkg/providers/gcp/membrane.go @CGO_ENABLED=0 go build -o bin/membrane-aws -ldflags="-extldflags=-static" ./pkg/providers/aws/membrane.go + @CGO_ENABLED=0 go build -o bin/membrane-azure -ldflags="-extldflags=-static" ./pkg/providers/azure/membrane.go @CGO_ENABLED=0 go build -o bin/membrane-do -ldflags="-extldflags=-static" ./pkg/providers/do/membrane.go @CGO_ENABLED=0 go build -o bin/membrane-dev -ldflags="-extldflags=-static" ./pkg/providers/dev/membrane.go @@ -199,7 +193,20 @@ build-all-binaries: clean generate-proto # generate mock implementations generate-mocks: @echo Generating Mock Clients - @mkdir -p mocks/mock_secret_manager + @mkdir -p mocks/secret_manager @mkdir -p mocks/secrets_manager - @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/mock_secret_manager/mock.go - @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go \ No newline at end of file + @mkdir -p mocks/key_vault + @mkdir -p mocks/s3 + @mkdir -p mocks/sqs + @mkdir -p mocks/azblob + @mkdir -p mocks/mock_event_grid + @mkdir -p mocks/azqueue + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/secret_manager SecretManagerClient > mocks/secret_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface SecretsManagerAPI > mocks/secrets_manager/mock.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/storage/azblob/iface AzblobServiceUrlIface,AzblobContainerUrlIface,AzblobBlockBlobUrlIface,AzblobDownloadResponse > mocks/azblob/mock.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault KeyVaultClient > mocks/key_vault/mock.go + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/s3/s3iface S3API > mocks/s3/mock.go + @go run github.com/golang/mock/mockgen github.com/aws/aws-sdk-go/service/sqs/sqsiface SQSAPI > mocks/sqs/mock.go + @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi BaseClientAPI > mocks/mock_event_grid/mock.go + @go run github.com/golang/mock/mockgen github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi TopicsClientAPI > mocks/mock_event_grid/topic.go + @go run github.com/golang/mock/mockgen github.com/nitric-dev/membrane/pkg/plugins/queue/azqueue/iface AzqueueServiceUrlIface,AzqueueQueueUrlIface,AzqueueMessageUrlIface,AzqueueMessageIdUrlIface,DequeueMessagesResponseIface > mocks/azqueue/mock.go \ No newline at end of file diff --git a/pkg/adapters/grpc/errors.go b/pkg/adapters/grpc/errors.go index 46c72bab7..385184cf7 100644 --- a/pkg/adapters/grpc/errors.go +++ b/pkg/adapters/grpc/errors.go @@ -15,6 +15,10 @@ package grpc import ( + "fmt" + "reflect" + + v1 "github.com/nitric-dev/membrane/interfaces/nitric/v1" "github.com/nitric-dev/membrane/pkg/plugins/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -23,17 +27,109 @@ import ( // Provides GRPC error reporting func NewGrpcError(operation string, err error) error { if pe, ok := err.(*errors.PluginError); ok { - return newGrpcErrorWithCode(codes.Code(errors.Code(pe)), operation, pe) + code := codes.Code(errors.Code(pe)) + + ed := &v1.ErrorDetails{} + ed.Message = pe.Msg + if pe.Cause != nil { + ed.Cause = pe.Cause.Error() + } + ed.Scope = &v1.ErrorScope{ + Service: operation, + Plugin: pe.Plugin, + } + if len(pe.Args) > 0 { + args := make(map[string]string) + for k, v := range pe.Args { + args[k] = LogArg(v) + } + ed.Scope.Args = args + } + + s := status.New(code, pe.Msg) + s, _ = s.WithDetails(ed) + + return s.Err() + } else { return newGrpcErrorWithCode(codes.Internal, operation, err) } } func newGrpcErrorWithCode(code codes.Code, operation string, err error) error { - return status.Errorf(code, "%s: %v", operation, err) + ed := &v1.ErrorDetails{} + ed.Message = err.Error() + ed.Scope = &v1.ErrorScope{ + Service: operation, + } + + s := status.New(code, err.Error()) + s, _ = s.WithDetails(ed) + + return s.Err() } // Provides generic error for unregistered plugins func NewPluginNotRegisteredError(plugin string) error { - return status.Errorf(codes.Unimplemented, "%s plugin not registered", plugin) + ed := &v1.ErrorDetails{} + ed.Message = fmt.Sprintf("%s plugin not registered", plugin) + + s := status.New(codes.Unimplemented, ed.Message) + s, _ = s.WithDetails(ed) + + return s.Err() +} + +func LogArg(arg interface{}) string { + value := getValue(arg) + + if value.Kind() == reflect.Struct { + + str := "{" + for i := 0; i < value.NumField(); i++ { + + fieldType := value.Type().Field(i) + tag := fieldType.Tag.Get("log") + if tag == "" || tag == "-" { + continue + } + + if len(str) > 1 { + str += ", " + } + + field := value.Field(i) + str += fieldType.Name + ": " + LogArg(field.Interface()) + } + str += "}" + + return str + + } else if value.Kind() == reflect.Map { + str := "{" + + for k, v := range arg.(map[string]interface{}) { + if len(str) > 1 { + str += ", " + } + str += fmt.Sprintf("%v", k) + ": " + LogArg(v) + } + + str += "}" + + return str + + } else { + return fmt.Sprintf("%v", arg) + } +} + +func getValue(x interface{}) reflect.Value { + val := reflect.ValueOf(x) + + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + + return val } diff --git a/pkg/adapters/grpc/errors_test.go b/pkg/adapters/grpc/errors_test.go index 9497249f8..ed4a705f9 100644 --- a/pkg/adapters/grpc/errors_test.go +++ b/pkg/adapters/grpc/errors_test.go @@ -24,25 +24,50 @@ import ( . "github.com/onsi/gomega" ) +type SecretValue struct { + Type string `log:"Type"` + Factor int `log:"-"` + Value string +} + +type Secret struct { + Name string `json:"Name" log:"Name"` + Version string `json:"Version" log:"Version"` + Value *SecretValue `json:"Value" log:"Value"` +} + var _ = Describe("GRPC Errors", func() { Context("GrpcError", func() { When("plugin.errors.InvalidArgument", func() { It("Should report GRPC IllegalArgument error", func() { - newErr := errors.ErrorsWithScope("test") + newErr := errors.ErrorsWithScope("test", nil) + err := newErr( + codes.InvalidArgument, + "bad param", + nil, + ) + grpcErr := grpc.NewGrpcError("BadServer.BadCall", err) + Expect(grpcErr.Error()).To(ContainSubstring("rpc error: code = InvalidArgument desc = bad param")) + }) + }) + When("plugin.errors.InvalidArgument args", func() { + It("Should report GRPC IllegalArgument error with args", func() { + args := map[string]interface{}{"key": "value"} + newErr := errors.ErrorsWithScope("test", args) err := newErr( codes.InvalidArgument, "bad param", nil, ) grpcErr := grpc.NewGrpcError("BadServer.BadCall", err) - Expect(grpcErr.Error()).To(ContainSubstring("rpc error: code = InvalidArgument desc = BadServer.BadCall: test([]): bad param")) + Expect(grpcErr.Error()).To(ContainSubstring("rpc error: code = InvalidArgument desc = bad param")) }) }) When("Standard Error", func() { It("Should report GRPC Internal error", func() { err := fmt.Errorf("internal error") err = grpc.NewGrpcError("BadServer.BadCall", err) - Expect(err.Error()).To(ContainSubstring("rpc error: code = Internal desc = BadServer.BadCall: internal error")) + Expect(err.Error()).To(ContainSubstring("rpc error: code = Internal desc = internal error")) }) }) }) @@ -55,4 +80,70 @@ var _ = Describe("GRPC Errors", func() { }) }) }) + + Context("Logging Arg", func() { + When("string", func() { + It("return string value", func() { + Expect(grpc.LogArg("string")).To(BeEquivalentTo("string")) + }) + }) + + When("int", func() { + It("return string value", func() { + Expect(grpc.LogArg(123)).To(BeEquivalentTo("123")) + }) + }) + + When("bool", func() { + It("return string value", func() { + Expect(grpc.LogArg(true)).To(BeEquivalentTo("true")) + }) + }) + + When("float", func() { + It("return string value", func() { + Expect(grpc.LogArg(3.1415)).To(BeEquivalentTo("3.1415")) + }) + }) + + When("struct", func() { + It("return string value", func() { + + data := Secret{ + Name: "name", + Version: "3", + Value: &SecretValue{ + Type: "key", + Factor: 2, + Value: "2a4wijgPq0PpwJ76IjT7&lTBZ$5SGRcq", + }, + } + + value := grpc.LogArg(data) + Expect(value).To(BeEquivalentTo("{Name: name, Version: 3, Value: {Type: key}}")) + }) + }) + + When("map", func() { + It("return string value", func() { + secret := Secret{ + Name: "name", + Version: "3", + Value: &SecretValue{ + Type: "key", + Factor: 2, + Value: "2a4wijgPq0PpwJ76IjT7&lTBZ$5SGRcq", + }, + } + + valueMap := map[string]interface{}{ + "key": "value", + "secret": secret, + } + value := grpc.LogArg(valueMap) + Expect(value).To(ContainSubstring("secret: {Name: name, Version: 3, Value: {Type: key}}")) + Expect(value).To(ContainSubstring("key: value")) + }) + }) + }) }) diff --git a/pkg/adapters/grpc/event_grpc_test.go b/pkg/adapters/grpc/event_grpc_test.go index 9cb3d4998..83f0ae400 100644 --- a/pkg/adapters/grpc/event_grpc_test.go +++ b/pkg/adapters/grpc/event_grpc_test.go @@ -16,7 +16,6 @@ package grpc_test import ( "context" - "fmt" "github.com/nitric-dev/membrane/pkg/adapters/grpc" @@ -36,7 +35,6 @@ type MockEventService struct { } func (m *MockEventService) Publish(topic string, event *events.NitricEvent) error { - fmt.Printf("Publish called %v", event) m.PublishTopic = topic m.PublishEvent = event return m.PublishError diff --git a/pkg/adapters/grpc/storage_grpc.go b/pkg/adapters/grpc/storage_grpc.go index 4d3d818e3..c86e23a8a 100644 --- a/pkg/adapters/grpc/storage_grpc.go +++ b/pkg/adapters/grpc/storage_grpc.go @@ -16,6 +16,7 @@ package grpc import ( "context" + "fmt" pb "github.com/nitric-dev/membrane/interfaces/nitric/v1" "github.com/nitric-dev/membrane/pkg/plugins/storage" @@ -86,6 +87,39 @@ func (s *StorageServiceServer) Delete(ctx context.Context, req *pb.StorageDelete } } +func convertOperation(operation pb.StoragePreSignUrlRequest_Operation) (storage.Operation, error) { + if operation == pb.StoragePreSignUrlRequest_READ { + return storage.READ, nil + } else if operation == pb.StoragePreSignUrlRequest_WRITE { + return storage.WRITE, nil + } + return 0, fmt.Errorf("unknown storage operation, supported operations are READ and WRITE") +} + +func (s *StorageServiceServer) PreSignUrl(ctx context.Context, req *pb.StoragePreSignUrlRequest) (*pb.StoragePreSignUrlResponse, error) { + if err := s.checkPluginRegistered(); err != nil { + return nil, err + } + + if err := req.ValidateAll(); err != nil { + return nil, newGrpcErrorWithCode(codes.InvalidArgument, "StorageService.PreSignUrl", err) + } + + intendedOp, err := convertOperation(req.GetOperation()) + // For safety, don't set a default operation (like read). Only perform known operations + if err != nil { + return nil, newGrpcErrorWithCode(codes.InvalidArgument, "StorageService.PreSignUrl", err) + } + + if url, err := s.storagePlugin.PreSignUrl(req.GetBucketName(), req.GetKey(), intendedOp, req.GetExpiry()); err == nil { + return &pb.StoragePreSignUrlResponse{ + Url: url, + }, nil + } else { + return nil, NewGrpcError("StorageService.PreSignUrl", err) + } +} + func NewStorageServiceServer(storagePlugin storage.StorageService) pb.StorageServiceServer { return &StorageServiceServer{ storagePlugin: storagePlugin, diff --git a/pkg/membrane/membrane.go b/pkg/membrane/membrane.go index 5f758a00a..21cf3a8e7 100644 --- a/pkg/membrane/membrane.go +++ b/pkg/membrane/membrane.go @@ -109,7 +109,7 @@ func (s *Membrane) log(log string) { } } -func (s *Membrane) CreateSecretServer() v1.SecretServiceServer { +func (s *Membrane) createSecretServer() v1.SecretServiceServer { return grpc2.NewSecretServer(s.secretPlugin) } @@ -119,7 +119,7 @@ func (s *Membrane) createDocumentServer() v1.DocumentServiceServer { } // Create a new Nitric events Server -func (s *Membrane) createeventsServer() v1.EventServiceServer { +func (s *Membrane) createEventsServer() v1.EventServiceServer { return grpc2.NewEventServiceServer(s.eventsPlugin) } @@ -162,14 +162,11 @@ func (s *Membrane) Start() error { var opts []grpc.ServerOption s.grpcServer = grpc.NewServer(opts...) - secretServer := s.CreateSecretServer() - v1.RegisterSecretServiceServer(s.grpcServer, secretServer) - // Load & Register the GRPC service plugins documentServer := s.createDocumentServer() v1.RegisterDocumentServiceServer(s.grpcServer, documentServer) - eventsServer := s.createeventsServer() + eventsServer := s.createEventsServer() v1.RegisterEventServiceServer(s.grpcServer, eventsServer) topicServer := s.createTopicServer() @@ -181,6 +178,9 @@ func (s *Membrane) Start() error { queueServer := s.createQueueServer() v1.RegisterQueueServiceServer(s.grpcServer, queueServer) + secretServer := s.createSecretServer() + v1.RegisterSecretServiceServer(s.grpcServer, secretServer) + // FaaS server MUST start before the child process if s.mode == Mode_Faas { faasServer := grpc2.NewFaasServer(s.pool) diff --git a/pkg/plugins/document/boltdb/boltdb.go b/pkg/plugins/document/boltdb/boltdb.go index 26554516f..d4423495f 100644 --- a/pkg/plugins/document/boltdb/boltdb.go +++ b/pkg/plugins/document/boltdb/boltdb.go @@ -16,6 +16,7 @@ package boltdb_service import ( "fmt" + "io" "os" "path/filepath" "strconv" @@ -59,7 +60,9 @@ func (d BoltDoc) String() string { func (s *BoltDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "BoltDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -105,7 +108,9 @@ func (s *BoltDocService) Get(key *document.Key) (*document.Document, error) { func (s *BoltDocService) Set(key *document.Key, content map[string]interface{}) error { newErr := errors.ErrorsWithScope( "BoltDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -151,7 +156,9 @@ func (s *BoltDocService) Set(key *document.Key, content map[string]interface{}) func (s *BoltDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "BoltDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -209,12 +216,7 @@ func (s *BoltDocService) Delete(key *document.Key) error { return nil } -func (s *BoltDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { - newErr := errors.ErrorsWithScope( - "BoltDocService.Query", - fmt.Sprintf("collection=%v", collection), - ) - +func (s *BoltDocService) query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string, newErr errors.ErrorFactory) (*document.QueryResult, error) { if err := document.ValidateQueryCollection(collection); err != nil { return nil, newErr( codes.InvalidArgument, @@ -351,6 +353,71 @@ func (s *BoltDocService) Query(collection *document.Collection, expressions []do }, nil } +func (s *BoltDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { + newErr := errors.ErrorsWithScope( + "BoltDocService.Query", + map[string]interface{}{ + "collection": collection, + }, + ) + + return s.query(collection, expressions, limit, pagingToken, newErr) +} + +func (s *BoltDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "BoltDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) + + var tmpLimit = limit + var documents []document.Document + var pagingToken map[string]string + + // Initial fetch + res, fetchErr := s.query(collection, expressions, limit, nil, newErr) + + if fetchErr != nil { + // Return an error only iterator if the initial fetch failed + return func() (*document.Document, error) { + return nil, fetchErr + } + } + + documents = res.Documents + pagingToken = res.PagingToken + + return func() (*document.Document, error) { + // check the iteration state + if tmpLimit == 0 && limit > 0 { + // we've reached the limit of reading + return nil, io.EOF + } else if pagingToken != nil && len(documents) == 0 { + // we've run out of documents and have more pages to read + res, fetchErr = s.query(collection, expressions, tmpLimit, pagingToken, newErr) + documents = res.Documents + pagingToken = res.PagingToken + } else if pagingToken == nil && len(documents) == 0 { + // we're all out of documents and pages before hitting the limit + return nil, io.EOF + } + + // We received an error fetching the docs + if fetchErr != nil { + return nil, fetchErr + } + + // pop the first element + var doc document.Document + doc, documents = documents[0], documents[1:] + tmpLimit = tmpLimit - 1 + + return &doc, nil + } +} + // New - Create a new dev KV plugin func New() (*BoltDocService, error) { dbDir := utils.GetEnv("LOCAL_DB_DIR", utils.GetRelativeDevPath(DEV_SUB_DIRECTORY)) diff --git a/pkg/plugins/document/dynamodb/dynamodb.go b/pkg/plugins/document/dynamodb/dynamodb.go index 31db0205e..9b6d608cb 100644 --- a/pkg/plugins/document/dynamodb/dynamodb.go +++ b/pkg/plugins/document/dynamodb/dynamodb.go @@ -16,6 +16,7 @@ package dynamodb_service import ( "fmt" + "io" "regexp" "sort" "strings" @@ -47,7 +48,9 @@ type DynamoDocService struct { func (s *DynamoDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "DynamoDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) err := document.ValidateKey(key) @@ -119,7 +122,9 @@ func (s *DynamoDocService) Get(key *document.Key) (*document.Document, error) { func (s *DynamoDocService) Set(key *document.Key, value map[string]interface{}) error { newErr := errors.ErrorsWithScope( "DynamoDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -175,7 +180,9 @@ func (s *DynamoDocService) Set(key *document.Key, value map[string]interface{}) func (s *DynamoDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "DynamoDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -230,7 +237,7 @@ func (s *DynamoDocService) Delete(key *document.Key) error { if err != nil { return newErr( codes.Internal, - fmt.Sprintf("error performing delete in table"), + "error performing delete in table", err, ) } @@ -241,7 +248,7 @@ func (s *DynamoDocService) Delete(key *document.Key) error { if err != nil { return newErr( codes.Internal, - fmt.Sprintf("error performing delete"), + "error performing delete", err, ) } @@ -255,10 +262,32 @@ func (s *DynamoDocService) Delete(key *document.Key) error { return nil } +func (s *DynamoDocService) query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { + queryResult := &document.QueryResult{ + Documents: make([]document.Document, 0), + } + + var resFunc resultRetriever = s.performQuery + if collection.Parent == nil || collection.Parent.Id == "" { + resFunc = s.performScan + } + + if res, err := resFunc(collection, expressions, limit, pagingToken); err != nil { + return nil, err + } else { + queryResult.Documents = append(queryResult.Documents, res.Documents...) + queryResult.PagingToken = res.PagingToken + } + + return queryResult, nil +} + func (s *DynamoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "DynamoDocService.Query", - fmt.Sprintf("collection=%v", collection), + map[string]interface{}{ + "collection": collection, + }, ) if err := document.ValidateQueryCollection(collection); err != nil { @@ -277,69 +306,116 @@ func (s *DynamoDocService) Query(collection *document.Collection, expressions [] ) } - queryResult := &document.QueryResult{ - Documents: make([]document.Document, 0), + queryResult, err := s.query(collection, expressions, limit, pagingToken) + if err != nil { + return nil, newErr( + codes.Internal, + "query error", + err, + ) } - // If partition key defined then perform a query - if collection.Parent != nil && collection.Parent.Id != "" { - err := s.performQuery(collection, expressions, limit, pagingToken, queryResult) - if err != nil { + remainingLimit := limit - len(queryResult.Documents) + + // If more results available, perform additional queries + for remainingLimit > 0 && + (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { + + if res, err := s.query(collection, expressions, remainingLimit, queryResult.PagingToken); err != nil { return nil, newErr( codes.Internal, "query error", err, ) + } else { + queryResult.Documents = append(queryResult.Documents, res.Documents...) + queryResult.PagingToken = res.PagingToken } - remainingLimit := limit - len(queryResult.Documents) + remainingLimit = limit - len(queryResult.Documents) + } + + return queryResult, nil +} - // If more results available, perform additional queries - for remainingLimit > 0 && - (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { +func (s *DynamoDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "DynamoDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) - err := s.performQuery(collection, expressions, remainingLimit, queryResult.PagingToken, queryResult) - if err != nil { - return nil, newErr( - codes.Internal, - "query error", - err, - ) - } + colErr := document.ValidateQueryCollection(collection) + expErr := document.ValidateExpressions(expressions) - remainingLimit = limit - len(queryResult.Documents) + if colErr != nil || expErr != nil { + // Return an error only iterator + return func() (*document.Document, error) { + return nil, newErr( + codes.InvalidArgument, + "invalid arguments", + fmt.Errorf("collection error:%v, expression error: %v", colErr, expErr), + ) } + } - } else { - err := s.performScan(collection, expressions, limit, pagingToken, queryResult) - if err != nil { + var tmpLimit = limit + var documents []document.Document + var pagingToken map[string]string + + // Initial fetch + res, fetchErr := s.query(collection, expressions, tmpLimit, nil) + + if fetchErr != nil { + // Return an error only iterator if the initial fetch failed + return func() (*document.Document, error) { return nil, newErr( codes.Internal, - "scan error", - err, + "query error", + fetchErr, ) } + } - remainingLimit := limit - len(queryResult.Documents) - - // If more results available, perform additional scans - for remainingLimit > 0 && - (queryResult.PagingToken != nil && len(queryResult.PagingToken) > 0) { + documents = res.Documents + pagingToken = res.PagingToken + + return func() (*document.Document, error) { + // check the iteration state + if tmpLimit <= 0 && limit > 0 { + // we've reached the limit of reading + return nil, io.EOF + } else if pagingToken != nil && len(documents) == 0 { + // we've run out of documents and have more pages to read + res, fetchErr = s.query(collection, expressions, tmpLimit, pagingToken) + documents = res.Documents + pagingToken = res.PagingToken + } else if pagingToken == nil && len(documents) == 0 { + // we're all out of documents and pages before hitting the limit + return nil, io.EOF + } - err := s.performScan(collection, expressions, remainingLimit, queryResult.PagingToken, queryResult) - if err != nil { - return nil, newErr( - codes.Internal, - "scan error", - err, - ) - } + // We received an error fetching the docs + if fetchErr != nil { + return nil, newErr( + codes.Internal, + "query error", + fetchErr, + ) + } - remainingLimit = limit - len(queryResult.Documents) + if len(documents) == 0 { + return nil, io.EOF } - } - return queryResult, nil + // pop the first element + var doc document.Document + doc, documents = documents[0], documents[1:] + tmpLimit = tmpLimit - 1 + + return &doc, nil + } } // New - Create a new DynamoDB key value plugin implementation @@ -407,16 +483,23 @@ func createItemMap(source map[string]interface{}, key *document.Key) map[string] return newMap } +type resultRetriever = func( + collection *document.Collection, + expressions []document.QueryExpression, + limit int, + pagingToken map[string]string, +) (*document.QueryResult, error) + func (s *DynamoDocService) performQuery( collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string, - queryResult *document.QueryResult) error { +) (*document.QueryResult, error) { if collection.Parent == nil { // Should never occur - return fmt.Errorf("cannot perform query without partion key defined") + return nil, fmt.Errorf("cannot perform query without partion key defined") } // Sort expressions to help map where "A >= %1 AND A <= %2" to DynamoDB expression "A BETWEEN %1 AND %2" @@ -425,7 +508,7 @@ func (s *DynamoDocService) performQuery( tableName, err := s.getTableName(*collection) if err != nil { - return err + return nil, err } input := &dynamodb.QueryInput{ @@ -462,7 +545,7 @@ func (s *DynamoDocService) performQuery( expKey := fmt.Sprintf(":%v%v", exp.Operand, i) valAttrib, err := dynamodbattribute.Marshal(exp.Value) if err != nil { - return fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) + return nil, fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) } input.ExpressionAttributeValues[expKey] = valAttrib } @@ -475,7 +558,7 @@ func (s *DynamoDocService) performQuery( if len(pagingToken) > 0 { startKey, err := dynamodbattribute.MarshalMap(pagingToken) if err != nil { - return fmt.Errorf("error performing query %v: %v", input, err) + return nil, fmt.Errorf("error performing query %v: %v", input, err) } input.SetExclusiveStartKey(startKey) } @@ -485,10 +568,10 @@ func (s *DynamoDocService) performQuery( resp, err := s.client.Query(input) if err != nil { - return fmt.Errorf("error performing query %v: %v", input, err) + return nil, fmt.Errorf("error performing query %v: %v", input, err) } - return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey, queryResult) + return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey) } func (s *DynamoDocService) performScan( @@ -496,12 +579,15 @@ func (s *DynamoDocService) performScan( expressions []document.QueryExpression, limit int, pagingToken map[string]string, - queryResult *document.QueryResult) error { +) (*document.QueryResult, error) { // Sort expressions to help map where "A >= %1 AND A <= %2" to DynamoDB expression "A BETWEEN %1 AND %2" sort.Sort(document.ExpsSort(expressions)) tableName, err := s.getTableName(*collection) + if err != nil { + return nil, err + } input := &dynamodb.ScanInput{ TableName: tableName, @@ -538,7 +624,7 @@ func (s *DynamoDocService) performScan( expKey := fmt.Sprintf(":%v%v", exp.Operand, i) valAttrib, err := dynamodbattribute.Marshal(exp.Value) if err != nil { - return fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) + return nil, fmt.Errorf("error marshalling %v: %v", exp.Operand, exp.Value) } input.ExpressionAttributeValues[expKey] = valAttrib } @@ -552,7 +638,7 @@ func (s *DynamoDocService) performScan( if len(pagingToken) > 0 { startKey, err := dynamodbattribute.MarshalMap(pagingToken) if err != nil { - return fmt.Errorf("error performing scan %v: %v", input, err) + return nil, fmt.Errorf("error performing scan %v: %v", input, err) } input.SetExclusiveStartKey(startKey) } @@ -561,21 +647,22 @@ func (s *DynamoDocService) performScan( resp, err := s.client.Scan(input) if err != nil { - return fmt.Errorf("error performing scan %v: %v", input, err) + return nil, fmt.Errorf("error performing scan %v: %v", input, err) } - return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey, queryResult) + return marshalQueryResult(collection, resp.Items, resp.LastEvaluatedKey) } -func marshalQueryResult(collection *document.Collection, items []map[string]*dynamodb.AttributeValue, lastEvaluatedKey map[string]*dynamodb.AttributeValue, queryResult *document.QueryResult) error { - +func marshalQueryResult(collection *document.Collection, items []map[string]*dynamodb.AttributeValue, lastEvaluatedKey map[string]*dynamodb.AttributeValue) (*document.QueryResult, error) { // Unmarshal Dynamo response items + var pTkn map[string]string = nil var valueMaps []map[string]interface{} - err := dynamodbattribute.UnmarshalListOfMaps(items, &valueMaps) - if err != nil { - return fmt.Errorf("error unmarshalling query response: %v", err) + if err := dynamodbattribute.UnmarshalListOfMaps(items, &valueMaps); err != nil { + return nil, fmt.Errorf("error unmarshalling query response: %v", err) } + docs := make([]document.Document, 0, len(valueMaps)) + // Strip keys & append results for _, m := range valueMaps { // Retrieve the original ID on the result @@ -614,20 +701,22 @@ func marshalQueryResult(collection *document.Collection, items []map[string]*dyn }, Content: m, } - queryResult.Documents = append(queryResult.Documents, sdkDoc) + docs = append(docs, sdkDoc) } // Unmarshal lastEvalutedKey var resultPagingToken map[string]string if len(lastEvaluatedKey) > 0 { - err = dynamodbattribute.UnmarshalMap(lastEvaluatedKey, &resultPagingToken) - if err != nil { - return fmt.Errorf("error unmarshalling query lastEvaluatedKey: %v", err) + if err := dynamodbattribute.UnmarshalMap(lastEvaluatedKey, &resultPagingToken); err != nil { + return nil, fmt.Errorf("error unmarshalling query lastEvaluatedKey: %v", err) } - queryResult.PagingToken = resultPagingToken + pTkn = resultPagingToken } - return nil + return &document.QueryResult{ + Documents: docs, + PagingToken: pTkn, + }, nil } func createFilterExpression(expressions []document.QueryExpression) string { diff --git a/pkg/plugins/document/firestore/firestore.go b/pkg/plugins/document/firestore/firestore.go index cb8c47818..aa5344842 100644 --- a/pkg/plugins/document/firestore/firestore.go +++ b/pkg/plugins/document/firestore/firestore.go @@ -17,6 +17,7 @@ package firestore_service import ( "context" "fmt" + "io" "strings" "github.com/nitric-dev/membrane/pkg/plugins/document" @@ -43,7 +44,9 @@ type FirestoreDocService struct { func (s *FirestoreDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "FirestoreDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -79,7 +82,9 @@ func (s *FirestoreDocService) Get(key *document.Key) (*document.Document, error) func (s *FirestoreDocService) Set(key *document.Key, value map[string]interface{}) error { newErr := errors.ErrorsWithScope( "FirestoreDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -114,7 +119,9 @@ func (s *FirestoreDocService) Set(key *document.Key, value map[string]interface{ func (s *FirestoreDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "FirestoreDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -179,10 +186,41 @@ func (s *FirestoreDocService) Delete(key *document.Key) error { return nil } +// +func (s *FirestoreDocService) buildQuery(collection *document.Collection, expressions []document.QueryExpression, limit int) (query firestore.Query, orderBy string) { + // Select correct root collection to perform query on + query = s.getQueryRoot(collection) + + for _, exp := range expressions { + expOperand := exp.Operand + if exp.Operator == "startsWith" { + expVal := fmt.Sprintf("%v", exp.Value) + endRangeValue := document.GetEndRangeValue(expVal) + query = query.Where(expOperand, ">=", exp.Value).Where(expOperand, "<", endRangeValue) + + } else { + query = query.Where(expOperand, exp.Operator, exp.Value) + } + + if exp.Operator != "==" && limit > 0 && orderBy == "" { + query = query.OrderBy(expOperand, firestore.Asc) + orderBy = expOperand + } + } + + if limit > 0 { + query = query.Limit(limit) + } + + return +} + func (s *FirestoreDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { newErr := errors.ErrorsWithScope( "FirestoreDocService.Query", - fmt.Sprintf("collection=%v", collection), + map[string]interface{}{ + "collection": collection, + }, ) if err := document.ValidateQueryCollection(collection); err != nil { @@ -206,41 +244,17 @@ func (s *FirestoreDocService) Query(collection *document.Collection, expressions } // Select correct root collection to perform query on - query := s.getQueryRoot(collection) - - var orderByAttrib string - - for _, exp := range expressions { - expOperand := exp.Operand - if exp.Operator == "startsWith" { - expVal := fmt.Sprintf("%v", exp.Value) - endRangeValue := document.GetEndRangeValue(expVal) - query = query.Where(expOperand, ">=", exp.Value).Where(expOperand, "<", endRangeValue) - - } else { - query = query.Where(expOperand, exp.Operator, exp.Value) - } - - if exp.Operator != "==" && limit > 0 && orderByAttrib == "" { - query = query.OrderBy(expOperand, firestore.Asc) - orderByAttrib = expOperand - } - } - - if limit > 0 { - query = query.Limit(limit) - - if len(pagingToken) > 0 { - query = query.OrderBy(firestore.DocumentID, firestore.Asc) + query, orderBy := s.buildQuery(collection, expressions, limit) - if tokens, ok := pagingToken[pagingTokens]; ok { - var vals []interface{} - for _, v := range strings.Split(tokens, "|") { - vals = append(vals, v) - } - query = query.StartAfter(vals...) + if len(pagingToken) > 0 { + query = query.OrderBy(firestore.DocumentID, firestore.Asc) + if tokens, ok := pagingToken[pagingTokens]; ok { + var vals []interface{} + for _, v := range strings.Split(tokens, "|") { + vals = append(vals, v) } + query = query.StartAfter(vals...) } } @@ -253,31 +267,15 @@ func (s *FirestoreDocService) Query(collection *document.Collection, expressions err, ) } - sdkDoc := document.Document{ - Content: docSnp.Data(), - Key: &document.Key{ - Collection: collection, - Id: docSnp.Ref.ID, - }, - } - - if p := docSnp.Ref.Parent.Parent; p != nil { - sdkDoc.Key.Collection = &document.Collection{ - Name: collection.Name, - Parent: &document.Key{ - Collection: collection.Parent.Collection, - Id: p.ID, - }, - } - } + sdkDoc := docSnpToDocument(collection, docSnp) queryResult.Documents = append(queryResult.Documents, sdkDoc) // If query limit configured determine continue tokens if limit > 0 && len(queryResult.Documents) == limit { tokens := "" - if orderByAttrib != "" { - tokens = fmt.Sprintf("%v", docSnp.Data()[orderByAttrib]) + "|" + if orderBy != "" { + tokens = fmt.Sprintf("%v", docSnp.Data()[orderBy]) + "|" } tokens += docSnp.Ref.ID @@ -290,6 +288,75 @@ func (s *FirestoreDocService) Query(collection *document.Collection, expressions return queryResult, nil } +func (s *FirestoreDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "FirestoreDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) + + colErr := document.ValidateQueryCollection(collection) + expErr := document.ValidateExpressions(expressions) + + if colErr != nil || expErr != nil { + // Return an error only iterator + return func() (*document.Document, error) { + return nil, newErr( + codes.InvalidArgument, + "invalid arguments", + fmt.Errorf("collection error:%v, expression error: %v", colErr, expErr), + ) + } + } + + query, _ := s.buildQuery(collection, expressions, limit) + + iter := query.Documents(s.context) + + return func() (*document.Document, error) { + docSnp, err := iter.Next() + + if err != nil { + if err == iterator.Done { + return nil, io.EOF + } + + return nil, newErr( + codes.Internal, + "error querying value", + err, + ) + } + + sdkDoc := docSnpToDocument(collection, docSnp) + + return &sdkDoc, nil + } +} + +func docSnpToDocument(col *document.Collection, snp *firestore.DocumentSnapshot) document.Document { + sdkDoc := document.Document{ + Content: snp.Data(), + Key: &document.Key{ + Collection: col, + Id: snp.Ref.ID, + }, + } + + if p := snp.Ref.Parent.Parent; p != nil { + sdkDoc.Key.Collection = &document.Collection{ + Name: col.Name, + Parent: &document.Key{ + Collection: col.Parent.Collection, + Id: p.ID, + }, + } + } + + return sdkDoc +} + func New() (document.DocumentService, error) { ctx := context.Background() diff --git a/pkg/plugins/document/mongodb/mongodb.go b/pkg/plugins/document/mongodb/mongodb.go index 3e99251b0..f09f4a38c 100644 --- a/pkg/plugins/document/mongodb/mongodb.go +++ b/pkg/plugins/document/mongodb/mongodb.go @@ -17,6 +17,7 @@ package mongodb_service import ( "context" "fmt" + "io" "strings" "time" @@ -37,8 +38,8 @@ const ( mongoDBSetDirectEnvVarName = "MONGODB_DIRECT" primaryKeyAttr = "_id" - parentKeyAttr = "_parent_id" - childrenAttr = "_child_colls" + parentKeyAttr = "_parent_id" + childrenAttr = "_child_colls" ) // Mapping to mongo operators, startsWith will be handled within the function @@ -51,11 +52,9 @@ var mongoOperatorMap = map[string]string{ ">": "$gt", } - - type MongoDocService struct { client *mongo.Client - db *mongo.Database + db *mongo.Database context context.Context document.UnimplementedDocumentPlugin } @@ -63,7 +62,9 @@ type MongoDocService struct { func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { newErr := errors.ErrorsWithScope( "MongoDocService.Get", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -77,7 +78,7 @@ func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { col := s.getCollection(key) docRef := bson.M{primaryKeyAttr: key.Id} - var value map[string]interface{}; + var value map[string]interface{} opts := options.FindOne() @@ -85,7 +86,7 @@ func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { opts.SetProjection(bson.M{primaryKeyAttr: 0, parentKeyAttr: 0, childrenAttr: 0}) err := col.FindOne(s.context, docRef, opts).Decode(&value) - + if err != nil { var code = codes.Internal if err == mongo.ErrNoDocuments { @@ -112,7 +113,9 @@ func (s *MongoDocService) Get(key *document.Key) (*document.Document, error) { func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) error { newErr := errors.ErrorsWithScope( "MongoDocService.Set", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -131,20 +134,18 @@ func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) e ) } - - coll := s.getCollection(key) value = mapKeys(key, value) - + opts := options.Update().SetUpsert(true) - + filter := bson.M{primaryKeyAttr: key.Id} update := bson.D{{"$set", value}} - - _, err := coll.UpdateOne(s.context, filter, update, opts); - + + _, err := coll.UpdateOne(s.context, filter, update, opts) + if err != nil { return newErr( codes.Internal, @@ -156,7 +157,7 @@ func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) e // add references if key.Collection.Parent != nil { err := s.updateChildReferences(key, coll.Name(), "$addToSet") - + if err != nil { return newErr( codes.Internal, @@ -172,7 +173,9 @@ func (s *MongoDocService) Set(key *document.Key, value map[string]interface{}) e func (s *MongoDocService) Delete(key *document.Key) error { newErr := errors.ErrorsWithScope( "MongoDocService.Delete", - fmt.Sprintf("key=%v", key), + map[string]interface{}{ + "key": key, + }, ) if err := document.ValidateKey(key); err != nil { @@ -187,8 +190,8 @@ func (s *MongoDocService) Delete(key *document.Key) error { filter := bson.M{primaryKeyAttr: key.Id} - opts := options.FindOneAndDelete().SetProjection(bson.M{ childrenAttr: 1, primaryKeyAttr: 0 }) - + opts := options.FindOneAndDelete().SetProjection(bson.M{childrenAttr: 1, primaryKeyAttr: 0}) + var deletedDocument map[string]interface{} // Delete document @@ -222,7 +225,7 @@ func (s *MongoDocService) Delete(key *document.Key) error { // clean references if none left if key.Collection.Parent != nil { err := s.updateChildReferences(key, coll.Name(), "$pull") - + if err != nil { return newErr( codes.Internal, @@ -231,34 +234,11 @@ func (s *MongoDocService) Delete(key *document.Key) error { ) } } - - return nil; -} -func (s *MongoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { - newErr := errors.ErrorsWithScope( - "MongoDocService.Query", - fmt.Sprintf("collection=%v", collection), - ) - - if err := document.ValidateQueryCollection(collection); err != nil { - return nil, newErr( - codes.InvalidArgument, - "invalid key", - err, - ) - } - - if err := document.ValidateExpressions(expressions); err != nil { - return nil, newErr( - codes.InvalidArgument, - "invalid expressions", - err, - ) - } - - var orderByAttrib string + return nil +} +func (s *MongoDocService) getCursor(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (cursor *mongo.Cursor, orderBy string, err error) { coll := s.getCollection(&document.Key{Collection: collection}) query := bson.M{} @@ -266,7 +246,7 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d opts := options.Find() opts.SetProjection(bson.M{childrenAttr: 0}) - + if limit > 0 { opts.SetLimit(int64(limit)) @@ -278,14 +258,12 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d for _, v := range strings.Split(tokens, "|") { vals = append(vals, v) } - + query[primaryKeyAttr] = bson.D{{"$gt", vals[len(vals)-1]}} } } } - - if collection.Parent != nil && collection.Parent.Id != "" { query[parentKeyAttr] = collection.Parent.Id } @@ -295,12 +273,12 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d if exp.Operator == "startsWith" { expVal := fmt.Sprintf("%v", exp.Value) endRangeValue := document.GetEndRangeValue(expVal) - + startsWith := bson.D{ {s.getOperator(">="), expVal}, {s.getOperator("<"), endRangeValue}, } - + query[expOperand] = startsWith } else { @@ -309,17 +287,38 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d } } - if exp.Operator != "==" && limit > 0 && orderByAttrib == "" { + if exp.Operator != "==" && limit > 0 && orderBy == "" { opts.SetSort(bson.D{{expOperand, 1}}) - orderByAttrib = expOperand + orderBy = expOperand } } - + + cursor, err = coll.Find(s.context, query, opts) + + return +} + +func (s *MongoDocService) Query(collection *document.Collection, expressions []document.QueryExpression, limit int, pagingToken map[string]string) (*document.QueryResult, error) { + newErr := errors.ErrorsWithScope( + "MongoDocService.Query", + map[string]interface{}{ + "collection": collection, + }, + ) + + if colErr, expErr := document.ValidateQueryCollection(collection), document.ValidateExpressions(expressions); colErr != nil || expErr != nil { + return nil, newErr( + codes.InvalidArgument, + "invalid arguments", + fmt.Errorf("collection: %v, expressions%v", colErr, expErr), + ) + } + queryResult := &document.QueryResult{ Documents: make([]document.Document, 0), } - cursor, err := coll.Find(s.context, query, opts) + cursor, orderBy, err := s.getCursor(collection, expressions, limit, pagingToken) if err != nil { return nil, newErr( @@ -331,62 +330,130 @@ func (s *MongoDocService) Query(collection *document.Collection, expressions []d defer cursor.Close(s.context) for cursor.Next(s.context) { - var docSnap map[string]interface{}; - - err := cursor.Decode(&docSnap) + sdkDoc, err := mongoDocToDocument(collection, cursor) if err != nil { return nil, newErr( codes.Internal, - "error querying value", + "error decoding mongo document", err, ) } - id := docSnap[primaryKeyAttr].(string) + queryResult.Documents = append(queryResult.Documents, *sdkDoc) - // remove id from content - delete(docSnap, primaryKeyAttr) + // If query limit configured determine continue tokens + if limit > 0 && len(queryResult.Documents) == limit { + tokens := "" + if orderBy != "" { + tokens = fmt.Sprintf("%v", sdkDoc.Content[orderBy]) + "|" + } + tokens += sdkDoc.Key.Id - sdkDoc := document.Document{ - Content: docSnap, - Key: &document.Key{ - Collection: collection, - Id: id, - }, + queryResult.PagingToken = map[string]string{ + "pagingTokens": tokens, + } } + } - if docSnap[parentKeyAttr] != nil { - parentId := docSnap[parentKeyAttr].(string) + return queryResult, nil +} - sdkDoc.Key.Collection = &document.Collection{ - Name: collection.Name, - Parent: &document.Key{ - Collection: collection.Parent.Collection, - Id: parentId, - }, - } +func (s *MongoDocService) QueryStream(collection *document.Collection, expressions []document.QueryExpression, limit int) document.DocumentIterator { + newErr := errors.ErrorsWithScope( + "MongoDocService.QueryStream", + map[string]interface{}{ + "collection": collection, + }, + ) + + colErr := document.ValidateQueryCollection(collection) + expErr := document.ValidateExpressions(expressions) - delete(docSnap, parentKeyAttr) + if colErr != nil || expErr != nil { + // Return an error only iterator + return func() (*document.Document, error) { + return nil, newErr( + codes.InvalidArgument, + "invalid arguments", + fmt.Errorf("collection error:%v, expression error: %v", colErr, expErr), + ) } - - queryResult.Documents = append(queryResult.Documents, sdkDoc) + } - // If query limit configured determine continue tokens - if limit > 0 && len(queryResult.Documents) == limit { - tokens := "" - if orderByAttrib != "" { - tokens = fmt.Sprintf("%v", docSnap[orderByAttrib]) + "|" + cursor, _, cursorErr := s.getCursor(collection, expressions, limit, nil) + + return func() (*document.Document, error) { + if cursorErr != nil { + return nil, cursorErr + } + + if cursor.Next(s.context) { + // return the next document + doc, err := mongoDocToDocument(collection, cursor) + + if err != nil { + return nil, newErr( + codes.Internal, + "error decoding mongo document", + err, + ) } - tokens += id - queryResult.PagingToken = map[string]string{ - "pagingTokens": tokens, + return doc, nil + } else { + // there was an error + // Close the cursor + cursor.Close(s.context) + + if cursor.Err() != nil { + return nil, newErr( + codes.Internal, + "mongo cursor error", + cursor.Err(), + ) + } else { + return nil, io.EOF } } } +} - return queryResult, nil +func mongoDocToDocument(coll *document.Collection, cursor *mongo.Cursor) (*document.Document, error) { + var docSnap map[string]interface{} + + if err := cursor.Decode(&docSnap); err != nil { + return nil, err + } + + id := docSnap[primaryKeyAttr].(string) + + // remove id from content + delete(docSnap, primaryKeyAttr) + + sdkDoc := document.Document{ + Content: docSnap, + Key: &document.Key{ + Collection: coll, + Id: id, + }, + } + + if docSnap[parentKeyAttr] != nil { + parentId := docSnap[parentKeyAttr].(string) + + sdkDoc.Key.Collection = &document.Collection{ + Name: coll.Name, + Parent: &document.Key{ + Collection: coll.Parent.Collection, + Id: parentId, + }, + } + + delete(docSnap, parentKeyAttr) + } + + return &sdkDoc, nil } func New() (document.DocumentService, error) { @@ -396,7 +463,6 @@ func New() (document.DocumentService, error) { return nil, fmt.Errorf("MongoDB missing environment variable: %v", mongoDBConnectionStringEnvVarName) } - database := utils.GetEnv(mongoDBDatabaseEnvVarName, "") if database == "" { @@ -415,7 +481,7 @@ func New() (document.DocumentService, error) { if clientError != nil { return nil, fmt.Errorf("mongodb error creating client: %v", clientError) } - + connectError := client.Connect(ctx) if connectError != nil { @@ -426,7 +492,7 @@ func New() (document.DocumentService, error) { return &MongoDocService{ client: client, - db: db, + db: db, context: context.Background(), }, nil } @@ -436,7 +502,7 @@ func NewWithClient(client *mongo.Client, database string, ctx context.Context) d return &MongoDocService{ client: client, - db: db, + db: db, context: ctx, } } @@ -449,7 +515,7 @@ func mapKeys(key *document.Key, source map[string]interface{}) map[string]interf newMap[key] = value } - parentKey := key.Collection.Parent + parentKey := key.Collection.Parent newMap[primaryKeyAttr] = key.Id @@ -463,11 +529,11 @@ func mapKeys(key *document.Key, source map[string]interface{}) map[string]interf func (s *MongoDocService) updateChildReferences(key *document.Key, subCollectionName string, action string) error { parentColl := s.getCollection(key.Collection.Parent) filter := bson.M{primaryKeyAttr: key.Collection.Parent.Id} - referenceMeta := bson.M{ childrenAttr: subCollectionName } + referenceMeta := bson.M{childrenAttr: subCollectionName} update := bson.D{{action, referenceMeta}} opts := options.Update().SetUpsert(true) - _, err := parentColl.UpdateOne(s.context, filter, update, opts); + _, err := parentColl.UpdateOne(s.context, filter, update, opts) if err != nil { return err @@ -476,7 +542,7 @@ func (s *MongoDocService) updateChildReferences(key *document.Key, subCollection return nil } -func (s *MongoDocService) getCollection(key *document.Key) *mongo.Collection { +func (s *MongoDocService) getCollection(key *document.Key) *mongo.Collection { collectionNames := []string{} parentKey := key.Collection.Parent @@ -492,4 +558,4 @@ func (s *MongoDocService) getCollection(key *document.Key) *mongo.Collection { func (s *MongoDocService) getOperator(operator string) string { return mongoOperatorMap[operator] -} \ No newline at end of file +} diff --git a/pkg/plugins/document/plugin.go b/pkg/plugins/document/plugin.go index 00025d38a..3aa00888e 100644 --- a/pkg/plugins/document/plugin.go +++ b/pkg/plugins/document/plugin.go @@ -23,17 +23,13 @@ import "fmt" const MaxSubCollectionDepth int = 1 type Collection struct { - Name string - Parent *Key + Name string `log:"Name"` + Parent *Key `log:"Parent"` } type Key struct { - Collection *Collection - Id string -} - -func (k *Key) String() string { - return fmt.Sprintf("Key{Collection: %v Id: %v}\n", k.Collection, k.Id) + Collection *Collection `log:"Collection"` + Id string `log:"Id"` } type Document struct { @@ -41,10 +37,6 @@ type Document struct { Content map[string]interface{} } -func (d *Document) String() string { - return fmt.Sprintf("Document{Content: %v}\n", d.Content) -} - type QueryExpression struct { Operand string Operator string diff --git a/pkg/plugins/errors/plugin_error.go b/pkg/plugins/errors/plugin_error.go index dfb55cfe1..71d7d2f09 100644 --- a/pkg/plugins/errors/plugin_error.go +++ b/pkg/plugins/errors/plugin_error.go @@ -21,59 +21,46 @@ import ( ) type PluginError struct { - code codes.Code - msg string - cause error + Code codes.Code + Msg string + Cause error + Plugin string + Args map[string]interface{} } func (p *PluginError) Unwrap() error { - return p.cause + return p.Cause } func (p *PluginError) Error() string { - if p.cause != nil { + if p.Cause != nil { // If the wrapped error is an ApiError than these should unwrap - return fmt.Sprintf("%s: \n %s", p.msg, p.cause.Error()) + return fmt.Sprintf("%s: \n %s", p.Msg, p.Cause.Error()) } - return fmt.Sprintf("%s", p.msg) + return fmt.Sprintf("%s", p.Msg) } // Code - returns a nitric api error code from an error or Unknown if the error was not a nitric api error func Code(e error) codes.Code { if pe, ok := e.(*PluginError); ok { - return pe.code + return pe.Code } return codes.Unknown } -// New - Creates a new nitric API error -func new(c codes.Code, msg string) error { - return &PluginError{ - code: c, - msg: msg, - } -} - -// NewWithCause - Creates a new nitric API error with the given error as it's cause -func newWithCause(c codes.Code, msg string, cause error) error { - return &PluginError{ - code: c, - msg: msg, - cause: cause, - } -} +type ErrorFactory = func(c codes.Code, msg string, cause error) error // ErrorsWithScope - Returns a new reusable error factory with the given scope -func ErrorsWithScope(s string, ctx ...interface{}) func(c codes.Code, msg string, cause error) error { - return func(c codes.Code, msg string, cause error) error { - sMsg := fmt.Sprintf("%s(%v): %s", s, ctx, msg) - - if cause == nil { - return new(c, sMsg) +func ErrorsWithScope(scope string, args map[string]interface{}) ErrorFactory { + return func(code codes.Code, msg string, cause error) error { + return &PluginError{ + Code: code, + Msg: msg, + Cause: cause, + Plugin: scope, + Args: args, } - - return newWithCause(c, sMsg, cause) } } diff --git a/pkg/plugins/events/dev/eventing.go b/pkg/plugins/events/dev/eventing.go index d80c662fb..d0c36dfb1 100644 --- a/pkg/plugins/events/dev/eventing.go +++ b/pkg/plugins/events/dev/eventing.go @@ -45,7 +45,10 @@ type LocalHttpeventsClient interface { func (s *LocalEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope( "LocalEventService.Publish", - fmt.Sprintf("topic=%s", topic), + map[string]interface{}{ + "topic": topic, + "event": event, + }, ) requestId := event.ID @@ -96,7 +99,7 @@ func (s *LocalEventService) Publish(topic string, event *events.NitricEvent) err } else { return newErr( codes.NotFound, - fmt.Sprintf("unable to find subscriber for topic"), + "unable to find subscriber for topic", nil, ) } diff --git a/pkg/plugins/events/event.go b/pkg/plugins/events/event.go index 33656a7f9..609c53ca8 100644 --- a/pkg/plugins/events/event.go +++ b/pkg/plugins/events/event.go @@ -15,7 +15,7 @@ package events // NitricEvent - An event for asynchronous processing and reactive programming type NitricEvent struct { - ID string `json:"id,omitempty"` - PayloadType string `json:"payloadType,omitempty"` + ID string `json:"id,omitempty" log:"ID"` + PayloadType string `json:"payloadType,omitempty" log:"PayloadType"` Payload map[string]interface{} `json:"payload,omitempty"` } diff --git a/pkg/plugins/events/eventgrid/eventgrid.go b/pkg/plugins/events/eventgrid/eventgrid.go new file mode 100644 index 000000000..a8d310545 --- /dev/null +++ b/pkg/plugins/events/eventgrid/eventgrid.go @@ -0,0 +1,209 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventgrid_service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid" + "github.com/Azure/azure-sdk-for-go/services/eventgrid/2018-01-01/eventgrid/eventgridapi" + eventgridmgmt "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid" + eventgridmgmtapi "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid/eventgridapi" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/date" + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" + "github.com/nitric-dev/membrane/pkg/plugins/events" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" + "github.com/nitric-dev/membrane/pkg/utils" +) + +type EventGridEventService struct { + events.UnimplementedeventsPlugin + client eventgridapi.BaseClientAPI + topicClient eventgridmgmtapi.TopicsClientAPI +} + +func (s *EventGridEventService) ListTopics() ([]string, error) { + newErr := errors.ErrorsWithScope( + "EventGrid.ListTopics", + map[string]interface{}{ + "list": "topics", + }, + ) + //Set the topic page length + pageLength := int32(10) + + ctx := context.Background() + results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) + + if err != nil { + return nil, newErr( + codes.Internal, + "error listing by subscription", + err, + ) + } + + var topics []string + + //Iterate over the topic pages adding their names to the topics slice + for results.NotDone() { + topicsList := results.Values() + for _, topic := range topicsList { + topics = append(topics, *topic.Name) + } + results.NextWithContext(ctx) + } + + return topics, nil +} + +func (s *EventGridEventService) getTopicEndpoint(topicName string) (string, error) { + ctx := context.Background() + pageLength := int32(10) + results, err := s.topicClient.ListBySubscription(ctx, "", &pageLength) + if err != nil { + return "", fmt.Errorf(err.Error()) + } + + for results.NotDone() { + topicsList := results.Values() + for _, topic := range topicsList { + if *topic.Name == topicName { + return strings.TrimSuffix(strings.TrimPrefix(*topic.Endpoint, "https://"), "/api/events"), nil + } + } + results.Next() + } + return "", fmt.Errorf("topic with provided name could not be found") +} + +func (s *EventGridEventService) nitricEventsToAzureEvents(topic string, events []*events.NitricEvent) ([]eventgrid.Event, error) { + var azureEvents []eventgrid.Event + for _, event := range events { + payload, err := json.Marshal(event.Payload) + if err != nil { + return nil, err + } + dataVersion := "1.0" + azureEvents = append(azureEvents, eventgrid.Event{ + ID: &event.ID, + Data: &payload, + EventType: &event.PayloadType, + Subject: &topic, + EventTime: &date.Time{time.Now()}, + DataVersion: &dataVersion, + }) + } + + return azureEvents, nil +} + +func (s *EventGridEventService) Publish(topic string, event *events.NitricEvent) error { + newErr := errors.ErrorsWithScope( + "EventGrid.Publish", + map[string]interface{}{ + "topic": topic, + }, + ) + ctx := context.Background() + + if len(topic) == 0 { + return newErr( + codes.InvalidArgument, + "provided invalid topic", + fmt.Errorf("non-blank topic is required"), + ) + } + if event == nil { + return newErr( + codes.InvalidArgument, + "provided invalid event", + fmt.Errorf("non-nil event is required"), + ) + } + + topicHostName, err := s.getTopicEndpoint(topic) + if err != nil { + return err + } + eventToPublish, err := s.nitricEventsToAzureEvents(topicHostName, []*events.NitricEvent{event}) + if err != nil { + return newErr( + codes.Internal, + "error marshalling event", + err, + ) + } + + result, err := s.client.PublishEvents(ctx, topicHostName, eventToPublish) + if err != nil { + return newErr( + codes.Internal, + "error publishing event", + err, + ) + } + + if result.StatusCode < 200 || result.StatusCode >= 300 { + return newErr( + codes.Internal, + "returned non 200 status code", + fmt.Errorf(result.Status), + ) + } + return nil +} + +func New() (events.EventService, error) { + subscriptionID := utils.GetEnv("AZURE_SUBSCRIPTION_ID", "") + if len(subscriptionID) == 0 { + return nil, fmt.Errorf("AZURE_SUBSCRIPTION_ID not configured") + } + + //Get the event grid token, using the event grid resource endpoint + spt, err := azureutils.GetServicePrincipalToken("https://eventgrid.azure.net") + if err != nil { + return nil, fmt.Errorf("error authenticating event grid client: %v", err.Error()) + } + //Get the event grid management token using the resource management endpoint + mgmtspt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceManagerEndpoint) + if err != nil { + return nil, fmt.Errorf("error authenticating event grid management client: %v", err.Error()) + } + client := eventgrid.New() + client.Authorizer = autorest.NewBearerAuthorizer(spt) + + topicClient := eventgridmgmt.NewTopicsClient(subscriptionID) + topicClient.Authorizer = autorest.NewBearerAuthorizer(mgmtspt) + + return &EventGridEventService{ + client: client, + topicClient: topicClient, + }, nil +} + +func NewWithClient(client eventgridapi.BaseClientAPI, topicClient eventgridmgmtapi.TopicsClientAPI) (events.EventService, error) { + return &EventGridEventService{ + client: client, + topicClient: topicClient, + }, nil +} diff --git a/pkg/plugins/events/eventgrid/eventgrid_suite_test.go b/pkg/plugins/events/eventgrid/eventgrid_suite_test.go new file mode 100644 index 000000000..08b9902ea --- /dev/null +++ b/pkg/plugins/events/eventgrid/eventgrid_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventgrid_service_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPubsub(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Event Grid Event Service Suite") +} diff --git a/pkg/plugins/events/eventgrid/eventgrid_test.go b/pkg/plugins/events/eventgrid/eventgrid_test.go new file mode 100644 index 000000000..2ddaef40e --- /dev/null +++ b/pkg/plugins/events/eventgrid/eventgrid_test.go @@ -0,0 +1,196 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventgrid_service_test + +import ( + "context" + "net/http" + + eventgridmgmt "github.com/Azure/azure-sdk-for-go/services/eventgrid/mgmt/2020-06-01/eventgrid" + "github.com/Azure/go-autorest/autorest" + "github.com/golang/mock/gomock" + mock_eventgrid "github.com/nitric-dev/membrane/mocks/mock_event_grid" + "github.com/nitric-dev/membrane/pkg/plugins/events" + eventgrid_service "github.com/nitric-dev/membrane/pkg/plugins/events/eventgrid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Event Grid Plugin", func() { + topicName := "Test" + topicEndpoint := "https://Test.local1-test.eventgrid.azure.net/api/events" + topicListResponsePage := eventgridmgmt.NewTopicsListResultPage( + eventgridmgmt.TopicsListResult{ + Value: &[]eventgridmgmt.Topic{ + { + Name: &topicName, + TopicProperties: &eventgridmgmt.TopicProperties{ + Endpoint: &topicEndpoint, + }, + }, + }, + }, + func(context.Context, eventgridmgmt.TopicsListResult) (eventgridmgmt.TopicsListResult, error) { + return eventgridmgmt.TopicsListResult{}, nil + }, + ) + When("Listing Available Topics", func() { + When("There are no topics available", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + Expect(topicClient).NotTo(BeNil()) + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(eventgridmgmt.TopicsListResultPage{}, nil).Times(1) + + It("Should return an empty list of topics", func() { + topics, err := eventgridPlugin.ListTopics() + Expect(err).To(BeNil()) + Expect(topics).To(BeEmpty()) + }) + }) + + When("There are topics available", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(topicListResponsePage, nil).Times(1) + + It("Should return all available topics", func() { + topics, err := eventgridPlugin.ListTopics() + Expect(err).To(BeNil()) + Expect(topics).To(ContainElement("Test")) + }) + }) + }) + + When("Publishing Messages", func() { + event := &events.NitricEvent{ + ID: "Test", + PayloadType: "Test", + Payload: map[string]interface{}{ + "Test": "Test", + }, + } + + When("To a topic that does not exist", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(eventgridmgmt.TopicsListResultPage{}, nil).Times(1) + + It("should return an error", func() { + err := eventgridPlugin.Publish("Test", event) + Expect(err).Should(HaveOccurred()) + }) + }) + + When("publishing to a topic that is unauthorised", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + eventgridClient.EXPECT().PublishEvents( + gomock.Any(), + "Test.local1-test.eventgrid.azure.net", + gomock.Any(), + ).Return(autorest.Response{ + &http.Response{ + StatusCode: 403, + }, + }, nil).Times(1) + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(topicListResponsePage, nil).Times(1) + + It("should return an error", func() { + err := eventgridPlugin.Publish("Test", event) + Expect(err).Should(HaveOccurred()) + }) + }) + + When("To a topic that does exist", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + eventgridClient.EXPECT().PublishEvents( + gomock.Any(), + "Test.local1-test.eventgrid.azure.net", + gomock.Any(), + ).Return(autorest.Response{ + &http.Response{ + StatusCode: 202, + }, + }, nil).Times(1) + topicClient.EXPECT().ListBySubscription( + gomock.Any(), + "", + gomock.Any(), + ).Return(topicListResponsePage, nil).Times(1) + + It("should successfully publish the message", func() { + err := eventgridPlugin.Publish("Test", event) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + When("Providing an empty topic", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + It("should return an error", func() { + err := eventgridPlugin.Publish("", event) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("provided invalid topic")) + }) + }) + + When("Providing an empty event", func() { + ctrl := gomock.NewController(GinkgoT()) + eventgridClient := mock_eventgrid.NewMockBaseClientAPI(ctrl) + topicClient := mock_eventgrid.NewMockTopicsClientAPI(ctrl) + eventgridPlugin, _ := eventgrid_service.NewWithClient(eventgridClient, topicClient) + + It("should return an error", func() { + err := eventgridPlugin.Publish("Test", nil) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("provided invalid event")) + }) + }) + }) +}) diff --git a/pkg/plugins/events/pubsub/pubsub.go b/pkg/plugins/events/pubsub/pubsub.go index 2eddae8f7..5b828fb21 100644 --- a/pkg/plugins/events/pubsub/pubsub.go +++ b/pkg/plugins/events/pubsub/pubsub.go @@ -35,7 +35,7 @@ type PubsubEventService struct { } func (s *PubsubEventService) ListTopics() ([]string, error) { - newErr := errors.ErrorsWithScope("PubsubEventService.ListTopics") + newErr := errors.ErrorsWithScope("PubsubEventService.ListTopics", nil) iter := s.client.Topics(context.TODO()) var topics []string @@ -57,7 +57,10 @@ func (s *PubsubEventService) ListTopics() ([]string, error) { func (s *PubsubEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope( "PubsubEventService.Publish", - fmt.Sprintf("topic=%s", topic), + map[string]interface{}{ + "topic": topic, + "event": event, + }, ) ctx := context.TODO() diff --git a/pkg/plugins/events/sns/sns.go b/pkg/plugins/events/sns/sns.go index febbc706b..f731c9dda 100644 --- a/pkg/plugins/events/sns/sns.go +++ b/pkg/plugins/events/sns/sns.go @@ -56,7 +56,10 @@ func (s *SnsEventService) getTopicArnFromName(name *string) (*string, error) { func (s *SnsEventService) Publish(topic string, event *events.NitricEvent) error { newErr := errors.ErrorsWithScope( "SnsEventService.Publish", - fmt.Sprintf("topic=%s", topic), + map[string]interface{}{ + "topic": topic, + "event": event, + }, ) data, err := json.Marshal(event) @@ -103,7 +106,7 @@ func (s *SnsEventService) Publish(topic string, event *events.NitricEvent) error } func (s *SnsEventService) ListTopics() ([]string, error) { - newErr := errors.ErrorsWithScope("SnsEventService.ListTopics") + newErr := errors.ErrorsWithScope("SnsEventService.ListTopics", nil) topicsOutput, err := s.client.ListTopics(&sns.ListTopicsInput{}) diff --git a/pkg/plugins/gateway/appservice/http_test.go b/pkg/plugins/gateway/appservice/http_test.go index 9a45c93d1..3d8c4964e 100644 --- a/pkg/plugins/gateway/appservice/http_test.go +++ b/pkg/plugins/gateway/appservice/http_test.go @@ -18,15 +18,16 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" - "github.com/nitric-dev/membrane/pkg/triggers" - "github.com/nitric-dev/membrane/pkg/worker" - mock_worker "github.com/nitric-dev/membrane/tests/mocks/worker" "io/ioutil" "net/http" "os" "time" + http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" + "github.com/nitric-dev/membrane/pkg/triggers" + "github.com/nitric-dev/membrane/pkg/worker" + mock_worker "github.com/nitric-dev/membrane/tests/mocks/worker" + "github.com/Azure/azure-sdk-for-go/profiles/latest/eventgrid/eventgrid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -88,11 +89,12 @@ var _ = Describe("Http", func() { When("With a SubscriptionValidation event", func() { It("Should return the provided validation code", func() { validationCode := "test" + payload := map[string]string{ + "ValidationCode": validationCode, + } evt := []eventgrid.Event{ { - Data: eventgrid.SubscriptionValidationEventData{ - ValidationCode: &validationCode, - }, + Data: payload, }, } diff --git a/pkg/plugins/gateway/lambda/lambda.go b/pkg/plugins/gateway/lambda/lambda.go index 9c71143b9..d3cc751f4 100644 --- a/pkg/plugins/gateway/lambda/lambda.go +++ b/pkg/plugins/gateway/lambda/lambda.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/url" "strings" "github.com/nitric-dev/membrane/pkg/triggers" @@ -137,14 +138,19 @@ func (event *Event) UnmarshalJSON(data []byte) error { // Copy the cookies over headerCopy["Cookie"] = evt.Cookies - event.Requests = append(event.Requests, &triggers.HttpRequest{ - // FIXME: Translate to http.Header - Header: headerCopy, - Body: []byte(evt.Body), - Method: evt.RequestContext.HTTP.Method, - Path: evt.RawPath, - Query: evt.QueryStringParameters, - }) + // Parse the raw query string + qVals, err := url.ParseQuery(evt.RawQueryString) + + if err == nil { + event.Requests = append(event.Requests, &triggers.HttpRequest{ + // FIXME: Translate to http.Header + Header: headerCopy, + Body: []byte(evt.Body), + Method: evt.RequestContext.HTTP.Method, + Path: evt.RawPath, + Query: qVals, + }) + } } break diff --git a/pkg/plugins/gateway/lambda/lambda_test.go b/pkg/plugins/gateway/lambda/lambda_test.go index 581ba6295..422283909 100644 --- a/pkg/plugins/gateway/lambda/lambda_test.go +++ b/pkg/plugins/gateway/lambda/lambda_test.go @@ -81,8 +81,9 @@ var _ = Describe("Lambda", func() { "x-nitric-request-id": "test-request-id", "Content-Type": "text/plain", }, - RawPath: "/test/test", - Body: "Test Payload", + RawPath: "/test/test", + RawQueryString: "key=test&key2=test1&key=test2", + Body: "Test Payload", RequestContext: events.APIGatewayV2HTTPRequestContext{ HTTP: events.APIGatewayV2HTTPRequestContextHTTPDescription{ Method: "GET", @@ -117,6 +118,10 @@ var _ = Describe("Lambda", func() { Expect(request.Method).To(Equal("GET")) By("Retaining the path") Expect(request.Path).To(Equal("/test/test")) + + By("Retaining the query parameters") + Expect(request.Query["key"]).To(BeEquivalentTo([]string{"test", "test2"})) + Expect(request.Query["key2"]).To(BeEquivalentTo([]string{"test1"})) }) }) }) diff --git a/pkg/plugins/queue/azqueue/azqueue.go b/pkg/plugins/queue/azqueue/azqueue.go new file mode 100644 index 000000000..63bfcd613 --- /dev/null +++ b/pkg/plugins/queue/azqueue/azqueue.go @@ -0,0 +1,289 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "time" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + azqueueserviceiface "github.com/nitric-dev/membrane/pkg/plugins/queue/azqueue/iface" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" + + "github.com/nitric-dev/membrane/pkg/utils" + + "github.com/Azure/azure-storage-queue-go/azqueue" + + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" + "github.com/nitric-dev/membrane/pkg/plugins/queue" +) + +// Set to 30 seconds, +const defaultVisibilityTimeout = 30 * time.Second + +type AzqueueQueueService struct { + client azqueueserviceiface.AzqueueServiceUrlIface +} + +// Returns an adapted azqueue MessagesUrl, which is a client for interacting with messages in a specific queue +func (s *AzqueueQueueService) getMessagesUrl(queue string) azqueueserviceiface.AzqueueMessageUrlIface { + qUrl := s.client.NewQueueURL(queue) + // Get a new messages URL (used to interact with messages in the queue) + return qUrl.NewMessageURL() +} + +// Returns an adapted azqueue MessageIdUrl, which is a client for interacting with a specific message (task) in a specific queue +func (s *AzqueueQueueService) getMessageIdUrl(queue string, messageId azqueue.MessageID) azqueueserviceiface.AzqueueMessageIdUrlIface { + mUrl := s.getMessagesUrl(queue) + + return mUrl.NewMessageIDURL(messageId) +} + +func (s *AzqueueQueueService) Send(queue string, task queue.NitricTask) error { + newErr := errors.ErrorsWithScope( + "AzqueueQueueService.Send", + map[string]interface{}{ + "queue": queue, + "task": task, + }, + ) + + messages := s.getMessagesUrl(queue) + + // Send the tasks to the queue + if taskBytes, err := json.Marshal(task); err == nil { + ctx := context.TODO() + if _, err := messages.Enqueue(ctx, string(taskBytes), 0, 0); err != nil { + return newErr( + codes.Internal, + "error sending task to queue", + err, + ) + } + } else { + return newErr( + codes.Internal, + "error marshalling the task", + err, + ) + } + + return nil +} + +func (s *AzqueueQueueService) SendBatch(queueName string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { + failedTasks := make([]*queue.FailedTask, 0) + + for _, task := range tasks { + // Azure Storage Queues don't support batches, so each task must be sent individually. + err := s.Send(queueName, task) + if err != nil { + failedTasks = append(failedTasks, &queue.FailedTask{ + Task: &task, + Message: err.Error(), + }) + } + } + + return &queue.SendBatchResponse{ + FailedTasks: failedTasks, + }, nil +} + +// AzureQueueItemLease - Represents a lease of an Azure Storages Queues item +// Azure requires a combination of their unique reference for a queue item (id) and a pop receipt (lease id) +// to perform operations on the item, such as delete it from the queue. +type AzureQueueItemLease struct { + // The ID of the queue item + // note: this is an id generated by Azure, it's not the user provided unique id. + ID string + // lease id, a new popReceipt is generated each time an item is dequeued. + PopReceipt string +} + +// String - convert the item lease struct to a string, to be returned as a NitricTask LeaseID +func (l *AzureQueueItemLease) String() (string, error) { + leaseID, err := json.Marshal(l) + return string(leaseID), err +} + +// leaseFromString - Unmarshal a NitricTask Lease ID (JSON) to an AzureQueueItemLease +func leaseFromString(leaseID string) (*AzureQueueItemLease, error) { + var lease AzureQueueItemLease + err := json.Unmarshal([]byte(leaseID), &lease) + if err != nil { + return nil, err + } + return &lease, nil +} + +// Receive - Receives a collection of tasks off a given queue. +func (s *AzqueueQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { + newErr := errors.ErrorsWithScope( + "AzqueueQueueService.Receive", + map[string]interface{}{ + "options": options, + }, + ) + + if err := options.Validate(); err != nil { + return nil, newErr( + codes.InvalidArgument, + "invalid receive options provided", + err, + ) + } + + messages := s.getMessagesUrl(options.QueueName) + + ctx := context.TODO() + dequeueResp, err := messages.Dequeue(ctx, int32(*options.Depth), defaultVisibilityTimeout) + if err != nil { + return nil, newErr( + codes.Internal, + "failed to received messages from the queue", + err, + ) + } + + if dequeueResp.NumMessages() == 0 { + return []queue.NitricTask{}, nil + } + + // Convert the Azure Storage Queues messages into Nitric tasks + var tasks []queue.NitricTask + for i := int32(0); i < dequeueResp.NumMessages(); i++ { + m := dequeueResp.Message(i) + var nitricTask queue.NitricTask + err := json.Unmarshal([]byte(m.Text), &nitricTask) + if err != nil { + // TODO: append error to error list and Nack the message. + continue + } + + lease := AzureQueueItemLease{ + ID: m.ID.String(), + PopReceipt: m.PopReceipt.String(), + } + leaseID, err := lease.String() + // This should never happen, it's a fatal error + if err != nil { + return nil, newErr( + codes.Internal, + "failed to construct queue item lease id", + err, + ) + } + + tasks = append(tasks, queue.NitricTask{ + ID: nitricTask.ID, + Payload: nitricTask.Payload, + PayloadType: nitricTask.PayloadType, + LeaseID: leaseID, + }) + } + + return tasks, nil +} + +// Complete - Completes a previously popped queue item +func (s *AzqueueQueueService) Complete(queue string, leaseId string) error { + newErr := errors.ErrorsWithScope( + "AzqueueQueueService.Complete", + map[string]interface{}{ + "queue": queue, + "leaseId": leaseId, + }, + ) + + lease, err := leaseFromString(leaseId) + if err != nil { + return newErr( + codes.InvalidArgument, + "failed to unmarshal lease id value", + err, + ) + } + + // Client for the specific message referenced by the lease + task := s.getMessageIdUrl(queue, azqueue.MessageID(lease.ID)) + ctx := context.TODO() + _, err = task.Delete(ctx, azqueue.PopReceipt(lease.PopReceipt)) + if err != nil { + return newErr( + codes.Internal, + "failed to complete task", + err, + ) + } + + return nil +} + +const expiryBuffer = 2 * time.Minute + +func tokenRefresherFromSpt(spt *adal.ServicePrincipalToken) azqueue.TokenRefresher { + return func(credential azqueue.TokenCredential) time.Duration { + if err := spt.Refresh(); err != nil { + fmt.Println("Error refreshing token: ", err) + } else { + tkn := spt.Token() + credential.SetToken(tkn.AccessToken) + + return tkn.Expires().Sub(time.Now().Add(expiryBuffer)) + } + + // Mark the token as already expired + return time.Duration(0) + } +} + +// New - Constructs a new Azure Storage Queues client with defaults +func New() (queue.QueueService, error) { + queueUrl := utils.GetEnv(azureutils.AZURE_STORAGE_QUEUE_ENDPOINT, "") + if queueUrl == "" { + return nil, fmt.Errorf("failed to determine Azure Storage Queue endpoint, environment variable %s not set", azureutils.AZURE_STORAGE_QUEUE_ENDPOINT) + } + + spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.Storage) + if err != nil { + return nil, err + } + + cTkn := azqueue.NewTokenCredential(spt.Token().AccessToken, tokenRefresherFromSpt(spt)) + + var accountURL *url.URL + if accountURL, err = url.Parse(queueUrl); err != nil { + return nil, err + } + + pipeline := azqueue.NewPipeline(cTkn, azqueue.PipelineOptions{}) + client := azqueue.NewServiceURL(*accountURL, pipeline) + + return &AzqueueQueueService{ + client: azqueueserviceiface.AdaptServiceUrl(client), + }, nil +} + +func NewWithClient(client azqueueserviceiface.AzqueueServiceUrlIface) queue.QueueService { + return &AzqueueQueueService{ + client: client, + } +} diff --git a/pkg/plugins/queue/azqueue/azqueue_suite_test.go b/pkg/plugins/queue/azqueue/azqueue_suite_test.go new file mode 100644 index 000000000..16f92c804 --- /dev/null +++ b/pkg/plugins/queue/azqueue/azqueue_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestAzqueue(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Azqueue Suite") +} diff --git a/pkg/plugins/queue/azqueue/azqueue_test.go b/pkg/plugins/queue/azqueue/azqueue_test.go new file mode 100644 index 000000000..36d6a7ca3 --- /dev/null +++ b/pkg/plugins/queue/azqueue/azqueue_test.go @@ -0,0 +1,368 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service + +import ( + "fmt" + "time" + + azqueue2 "github.com/Azure/azure-storage-queue-go/azqueue" + "github.com/nitric-dev/membrane/pkg/plugins/queue" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + mock_azqueue "github.com/nitric-dev/membrane/mocks/azqueue" +) + +var _ = Describe("Azqueue", func() { + + Context("Send", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).Times(1).Return(&azqueue2.EnqueueMessageResponse{}, nil) + + err := queuePlugin.Send("test-queue", queue.NitricTask{ + Payload: map[string]interface{}{"testval": "testkey"}, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).Times(1).Return(nil, fmt.Errorf("a test error")) + + err := queuePlugin.Send("test-queue", queue.NitricTask{ + Payload: map[string]interface{}{"testval": "testkey"}, + }) + + By("Returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) + + Context("Send Batch", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(2).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(2).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).Times(2).Return(&azqueue2.EnqueueMessageResponse{}, nil) + + resp, err := queuePlugin.SendBatch("test-queue", []queue.NitricTask{ + {Payload: map[string]interface{}{"testval": "testkey"}}, + {Payload: map[string]interface{}{"testval": "testkey"}}, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Not returning failed tasks") + Expect(len(resp.FailedTasks)).To(Equal(0)) + + crtl.Finish() + }) + }) + + When("Failing to send one task", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(2).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(2).Return(mockMessages) + + By("Calling Enqueue once on the Message URL with the expected options") + mockMessages.EXPECT().Enqueue( + gomock.Any(), + "{\"payload\":{\"testval\":\"testkey\"}}", + time.Duration(0), + time.Duration(0), + ).AnyTimes( /* Using AnyTimes because Times(2) doesn't work for multiple returns */ + ).Return(nil, fmt.Errorf("a test error")).Return(&azqueue2.EnqueueMessageResponse{}, nil) + + resp, err := queuePlugin.SendBatch("test-queue", []queue.NitricTask{ + {Payload: map[string]interface{}{"testval": "testkey"}}, + {Payload: map[string]interface{}{"testval": "testkey"}}, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Not returning failed tasks") + Expect(resp.FailedTasks).To(Equal([]*queue.FailedTask{})) + + crtl.Finish() + }) + }) + }) + + Context("Receive", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Dequeue once on the Message URL with the expected options") + mockMessages.EXPECT().Dequeue( + gomock.Any(), // ctx + int32(1), // depth + 30*time.Second, // visibility timeout - defaulted to 30 seconds + ).Times(1).Return(mockDequeueResp, nil) + + mockDequeueResp.EXPECT().NumMessages().AnyTimes().Return(int32(1)) + mockDequeueResp.EXPECT().Message(int32(0)).Times(1).Return(&azqueue2.DequeuedMessage{ + ID: "testid", + //InsertionTime: time.Time{}, + //ExpirationTime: time.Time{}, + PopReceipt: "popreceipt", + NextVisibleTime: time.Time{}, + DequeueCount: 0, + Text: "{\"payload\":{\"testval\":\"testkey\"}}", + }) + + depth := uint32(1) + + tasks, err := queuePlugin.Receive(queue.ReceiveOptions{ + QueueName: "test-queue", + Depth: &depth, + }) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Returning the dequeued task") + Expect(len(tasks)).To(Equal(1)) + Expect(tasks[0].Payload).To(Equal(map[string]interface{}{"testval": "testkey"})) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + //mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + By("Calling Dequeue once on the Message URL with the expected options") + mockMessages.EXPECT().Dequeue( + gomock.Any(), // ctx + int32(1), // depth + 30*time.Second, // visibility timeout - defaulted to 30 seconds + ).Times(1).Return(nil, fmt.Errorf("a test error")) + + depth := uint32(1) + + _, err := queuePlugin.Receive(queue.ReceiveOptions{ + QueueName: "test-queue", + Depth: &depth, + }) + + By("Returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) + + Context("Complete", func() { + When("Azure returns a successfully response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + lease := AzureQueueItemLease{ + ID: "testid", + PopReceipt: "testreceipt", + } + leaseStr, _ := lease.String() + + By("Retrieving the Message ID URL specific to the dequeued task") + mockMessages.EXPECT().NewMessageIDURL(azqueue2.MessageID("testid")).Times(1).Return(mockMessageId) + mockMessageId.EXPECT().Delete(gomock.Any(), azqueue2.PopReceipt(lease.PopReceipt)).Times(1).Return(nil, nil) + + err := queuePlugin.Complete("test-queue", leaseStr) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzqueue := mock_azqueue.NewMockAzqueueServiceUrlIface(crtl) + mockQueue := mock_azqueue.NewMockAzqueueQueueUrlIface(crtl) + mockMessages := mock_azqueue.NewMockAzqueueMessageUrlIface(crtl) + //mockDequeueResp := mock_azqueue.NewMockDequeueMessagesResponseIface(crtl) + mockMessageId := mock_azqueue.NewMockAzqueueMessageIdUrlIface(crtl) + + queuePlugin := &AzqueueQueueService{ + client: mockAzqueue, + } + + It("should successfully send the queue item(s)", func() { + By("Retrieving the Queue URL for the requested queue") + mockAzqueue.EXPECT().NewQueueURL("test-queue").Times(1).Return(mockQueue) + + By("Retrieving the Message URL of the requested queue") + mockQueue.EXPECT().NewMessageURL().Times(1).Return(mockMessages) + + lease := AzureQueueItemLease{ + ID: "testid", + PopReceipt: "testreceipt", + } + leaseStr, _ := lease.String() + + By("Retrieving the Message ID URL specific to the dequeued task") + mockMessages.EXPECT().NewMessageIDURL(azqueue2.MessageID("testid")).Times(1).Return(mockMessageId) + mockMessageId.EXPECT().Delete(gomock.Any(), azqueue2.PopReceipt(lease.PopReceipt)).Times(1).Return(nil, fmt.Errorf("a test error")) + + err := queuePlugin.Complete("test-queue", leaseStr) + + By("Returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) +}) diff --git a/pkg/plugins/queue/azqueue/iface/adapters.go b/pkg/plugins/queue/azqueue/iface/adapters.go new file mode 100644 index 000000000..fa64bd1b2 --- /dev/null +++ b/pkg/plugins/queue/azqueue/iface/adapters.go @@ -0,0 +1,88 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service_iface + +import ( + "context" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" +) + +func AdaptServiceUrl(c azqueue.ServiceURL) AzqueueServiceUrlIface { + return serviceUrl{c} +} + +func AdaptQueueUrl(c azqueue.QueueURL) AzqueueQueueUrlIface { + return queueUrl{c} +} + +func AdaptMessageUrl(c azqueue.MessagesURL) AzqueueMessageUrlIface { + return messageUrl{c} +} + +func AdaptMessageIdUrl(c azqueue.MessageIDURL) AzqueueMessageIdUrlIface { + return messageIdUrl{c} +} + +func AdaptDequeueMessagesResponse(c azqueue.DequeuedMessagesResponse) DequeueMessagesResponseIface { + return dequeueMessagesResponse{c} +} + +type ( + serviceUrl struct{ c azqueue.ServiceURL } + queueUrl struct{ c azqueue.QueueURL } + messageUrl struct{ c azqueue.MessagesURL } + messageIdUrl struct{ c azqueue.MessageIDURL } + dequeueMessagesResponse struct { + c azqueue.DequeuedMessagesResponse + } +) + +func (c serviceUrl) NewQueueURL(queueName string) AzqueueQueueUrlIface { + return AdaptQueueUrl(c.c.NewQueueURL(queueName)) +} + +func (c queueUrl) NewMessageURL() AzqueueMessageUrlIface { + return AdaptMessageUrl(c.c.NewMessagesURL()) +} + +func (c messageUrl) Enqueue(ctx context.Context, messageText string, visibilityTimeout time.Duration, timeToLive time.Duration) (*azqueue.EnqueueMessageResponse, error) { + return c.c.Enqueue(ctx, messageText, visibilityTimeout, timeToLive) +} + +func (c messageUrl) Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (DequeueMessagesResponseIface, error) { + resp, err := c.c.Dequeue(ctx, maxMessages, visibilityTimeout) + if err != nil { + return nil, err + } + return AdaptDequeueMessagesResponse(*resp), nil +} + +func (c messageUrl) NewMessageIDURL(messageId azqueue.MessageID) AzqueueMessageIdUrlIface { + return AdaptMessageIdUrl(c.c.NewMessageIDURL(messageId)) +} + +func (c messageIdUrl) Delete(ctx context.Context, popReceipt azqueue.PopReceipt) (*azqueue.MessageIDDeleteResponse, error) { + return c.c.Delete(ctx, popReceipt) +} + +func (c dequeueMessagesResponse) NumMessages() int32 { + return c.c.NumMessages() +} + +func (c dequeueMessagesResponse) Message(index int32) *azqueue.DequeuedMessage { + return c.c.Message(index) +} diff --git a/pkg/plugins/queue/azqueue/iface/iface.go b/pkg/plugins/queue/azqueue/iface/iface.go new file mode 100644 index 000000000..6ce2dfd58 --- /dev/null +++ b/pkg/plugins/queue/azqueue/iface/iface.go @@ -0,0 +1,45 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azqueue_service_iface + +import ( + "context" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" +) + +type AzqueueServiceUrlIface interface { + NewQueueURL(string) AzqueueQueueUrlIface +} + +type AzqueueQueueUrlIface interface { + NewMessageURL() AzqueueMessageUrlIface +} + +type AzqueueMessageUrlIface interface { + Enqueue(ctx context.Context, messageText string, visibilityTimeout time.Duration, timeToLive time.Duration) (*azqueue.EnqueueMessageResponse, error) + Dequeue(ctx context.Context, maxMessages int32, visibilityTimeout time.Duration) (DequeueMessagesResponseIface, error) + NewMessageIDURL(messageId azqueue.MessageID) AzqueueMessageIdUrlIface +} + +type AzqueueMessageIdUrlIface interface { + Delete(ctx context.Context, popReceipt azqueue.PopReceipt) (*azqueue.MessageIDDeleteResponse, error) +} + +type DequeueMessagesResponseIface interface { + NumMessages() int32 + Message(index int32) *azqueue.DequeuedMessage +} diff --git a/pkg/plugins/queue/dev/queue.go b/pkg/plugins/queue/dev/queue.go index e338417c7..c9e66b6c0 100644 --- a/pkg/plugins/queue/dev/queue.go +++ b/pkg/plugins/queue/dev/queue.go @@ -47,7 +47,10 @@ type Item struct { func (s *DevQueueService) Send(queue string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( "DevQueueService.Send", - fmt.Sprintf("queue=%s", queue), + map[string]interface{}{ + "queue": queue, + "task": task, + }, ) if queue == "" { @@ -96,7 +99,10 @@ func (s *DevQueueService) Send(queue string, task queue.NitricTask) error { func (s *DevQueueService) SendBatch(q string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { newErr := errors.ErrorsWithScope( "DevQueueService.SendBatch", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "tasks.len": len(tasks), + }, ) if q == "" { @@ -156,7 +162,9 @@ func (s *DevQueueService) SendBatch(q string, tasks []queue.NitricTask) (*queue. func (s *DevQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( "DevQueueService.Receive", - fmt.Sprintf("options=%v", options), + map[string]interface{}{ + "options": options, + }, ) if options.QueueName == "" { @@ -179,6 +187,13 @@ func (s *DevQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricT var items []Item err = db.All(&items, storm.Limit(int(*options.Depth))) + if err != nil { + return nil, newErr( + codes.Internal, + "error reading tasks", + err, + ) + } poppedTasks := make([]queue.NitricTask, 0) for _, item := range items { @@ -211,7 +226,10 @@ func (s *DevQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricT func (s *DevQueueService) Complete(queue string, leaseId string) error { newErr := errors.ErrorsWithScope( "DevQueueService.Complete", - fmt.Sprintf("queue=%s", queue), + map[string]interface{}{ + "queue": queue, + "leaseId": leaseId, + }, ) if queue == "" { diff --git a/pkg/plugins/queue/plugin.go b/pkg/plugins/queue/plugin.go index 8ee15bf39..f8fa8f96c 100644 --- a/pkg/plugins/queue/plugin.go +++ b/pkg/plugins/queue/plugin.go @@ -39,12 +39,12 @@ type ReceiveOptions struct { // Nitric name for the queue. // // queueName is a required field - QueueName string `type:"string" required:"true"` + QueueName string `type:"string" required:"true" log:"QueueName"` // Max depth of queue messages to receive from the queue. // // If nil or 0, defaults to depth 1. - Depth *uint32 `type:"int" required:"false"` + Depth *uint32 `type:"int" required:"false" log:"Depth"` } func (p *ReceiveOptions) Validate() error { @@ -77,6 +77,7 @@ type UnimplementedQueuePlugin struct { // Ensure UnimplementedQueuePlugin conforms to QueueService interface var _ QueueService = (*UnimplementedQueuePlugin)(nil) +// TODO: replace NitricTask and []NitricTask with pointers // Push - Unimplemented Stub for the UnimplementedQueuePlugin func (*UnimplementedQueuePlugin) Send(queue string, task NitricTask) error { return fmt.Errorf("UNIMPLEMENTED") diff --git a/pkg/plugins/queue/pubsub/pubsub.go b/pkg/plugins/queue/pubsub/pubsub.go index 81c9b37cd..e58b61c04 100644 --- a/pkg/plugins/queue/pubsub/pubsub.go +++ b/pkg/plugins/queue/pubsub/pubsub.go @@ -49,7 +49,10 @@ func generateQueueSubscription(queue string) string { func (s *PubsubQueueService) Send(queue string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( "PubsubQueueService.Send", - fmt.Sprintf("queue=%s", queue), + map[string]interface{}{ + "queue": queue, + "task": task, + }, ) // We'll be using pubsub with pull subscribers to facilitate queue functionality ctx := context.TODO() @@ -58,7 +61,7 @@ func (s *PubsubQueueService) Send(queue string, task queue.NitricTask) error { if exists, err := topic.Exists(ctx); !exists || err != nil { return newErr( codes.NotFound, - fmt.Sprintf("queue not found"), + "queue not found", err, ) } @@ -91,7 +94,10 @@ func (s *PubsubQueueService) Send(queue string, task queue.NitricTask) error { func (s *PubsubQueueService) SendBatch(q string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { newErr := errors.ErrorsWithScope( "PubsubQueueService.SendBatch", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "tasks.len": len(tasks), + }, ) // We'll be using pubsub with pull subscribers to facilitate queue functionality @@ -177,7 +183,9 @@ func (s *PubsubQueueService) getQueueSubscription(q string) (ifaces_pubsub.Subsc func (s *PubsubQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( "PubsubQueueService.Receive", - fmt.Sprintf("options=%v", options), + map[string]interface{}{ + "options": options, + }, ) if err := options.Validate(); err != nil { @@ -248,7 +256,7 @@ func (s *PubsubQueueService) Receive(options queue.ReceiveOptions) ([]queue.Nitr ID: nitricTask.ID, Payload: nitricTask.Payload, PayloadType: nitricTask.PayloadType, - LeaseID: nitricTask.LeaseID, + LeaseID: m.AckId, }) } @@ -259,7 +267,10 @@ func (s *PubsubQueueService) Receive(options queue.ReceiveOptions) ([]queue.Nitr func (s *PubsubQueueService) Complete(q string, leaseId string) error { newErr := errors.ErrorsWithScope( "PubsubQueueService.Complete", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "leaseId": leaseId, + }, ) ctx := context.Background() diff --git a/pkg/plugins/queue/sqs/sqs.go b/pkg/plugins/queue/sqs/sqs.go index 63b673e1a..46f3d5075 100644 --- a/pkg/plugins/queue/sqs/sqs.go +++ b/pkg/plugins/queue/sqs/sqs.go @@ -17,7 +17,6 @@ package sqs_service import ( "encoding/json" "fmt" - "strings" "github.com/nitric-dev/membrane/pkg/plugins/errors" "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" @@ -25,38 +24,63 @@ import ( "github.com/nitric-dev/membrane/pkg/utils" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" ) +const ( + // ErrCodeNoSuchTagSet - AWS API neglects to include a constant for this error code. + ErrCodeNoSuchTagSet = "NoSuchTagSet" +) + type SQSQueueService struct { queue.UnimplementedQueuePlugin client sqsiface.SQSAPI } // Get the URL for a given queue name -func (s *SQSQueueService) getUrlForQueueName(queueName string) (*string, error) { - // TODO: Need to be able to guarantee same accound deployment in this case - // In this case it would be preferred to use this method - // s.client.GetQueueUrl(&sqs.GetQueueUrlInput{}) - if out, err := s.client.ListQueues(&sqs.ListQueuesInput{}); err == nil { - for _, url := range out.QueueUrls { - if strings.HasSuffix(*url, queueName) { - return url, nil +func (s *SQSQueueService) getUrlForQueueName(queue string) (*string, error) { + out, err := s.client.ListQueues(&sqs.ListQueuesInput{}) + + if err != nil { + return nil, fmt.Errorf("Encountered an error retrieving the queue list: %v", err) + } + + for _, q := range out.QueueUrls { + // TODO: This could be rather slow, it's interesting that they don't return this in the list queues output + tagout, err := s.client.ListQueueTags(&sqs.ListQueueTagsInput{ + QueueUrl: q, + }) + + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == ErrCodeNoSuchTagSet { + // Ignore queues with no tags, check the next queue + continue + } + return nil, err } + return nil, err } - } else { - return nil, fmt.Errorf("An Unexpected error occurred: %s", err) - } - return nil, fmt.Errorf("Could not find Queue: %s", queueName) + for k, v := range tagout.Tags { + if k == "x-nitric-name" && *v == queue { + return q, nil + } + } + } + return nil, fmt.Errorf("Unable to find queue with name: %s", queue) } func (s *SQSQueueService) Send(queueName string, task queue.NitricTask) error { newErr := errors.ErrorsWithScope( "SQSQueueService.Send", - fmt.Sprintf("queue=%s", queueName), + map[string]interface{}{ + "queue": queueName, + "task": task, + }, ) tasks := []queue.NitricTask{task} @@ -73,7 +97,10 @@ func (s *SQSQueueService) Send(queueName string, task queue.NitricTask) error { func (s *SQSQueueService) SendBatch(queueName string, tasks []queue.NitricTask) (*queue.SendBatchResponse, error) { newErr := errors.ErrorsWithScope( "SQSQueueService.SendBatch", - fmt.Sprintf("queue=%s", queueName), + map[string]interface{}{ + "queue": queueName, + "tasks.len": len(tasks), + }, ) if url, err := s.getUrlForQueueName(queueName); err == nil { @@ -137,7 +164,9 @@ func (s *SQSQueueService) SendBatch(queueName string, tasks []queue.NitricTask) func (s *SQSQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricTask, error) { newErr := errors.ErrorsWithScope( "SQSQueueService.Receive", - fmt.Sprintf("options=%v", options), + map[string]interface{}{ + "options": options, + }, ) if err := options.Validate(); err != nil { @@ -205,7 +234,10 @@ func (s *SQSQueueService) Receive(options queue.ReceiveOptions) ([]queue.NitricT func (s *SQSQueueService) Complete(q string, leaseId string) error { newErr := errors.ErrorsWithScope( "SQSQueueService.Complete", - fmt.Sprintf("queue=%s", q), + map[string]interface{}{ + "queue": q, + "leaseId": leaseId, + }, ) if url, err := s.getUrlForQueueName(q); err == nil { diff --git a/pkg/plugins/queue/sqs/sqs_suite_test.go b/pkg/plugins/queue/sqs/sqs_suite_test.go index 58f598dbf..ab57c3874 100644 --- a/pkg/plugins/queue/sqs/sqs_suite_test.go +++ b/pkg/plugins/queue/sqs/sqs_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqs_service_test +package sqs_service import ( "testing" diff --git a/pkg/plugins/queue/sqs/sqs_test.go b/pkg/plugins/queue/sqs/sqs_test.go index a253c6eb2..e55ee469f 100644 --- a/pkg/plugins/queue/sqs/sqs_test.go +++ b/pkg/plugins/queue/sqs/sqs_test.go @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqs_service_test +package sqs_service import ( - "encoding/json" "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/events" - sqs_service "github.com/nitric-dev/membrane/pkg/plugins/queue/sqs" - mocks_sqs "github.com/nitric-dev/membrane/tests/mocks/sqs" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" + "github.com/golang/mock/gomock" + mocks_sqs "github.com/nitric-dev/membrane/mocks/sqs" "github.com/nitric-dev/membrane/pkg/plugins/queue" . "github.com/onsi/ginkgo" @@ -28,16 +28,112 @@ import ( ) var _ = Describe("Sqs", func() { - // Tests for the BatchPush method - Context("BatchPush", func() { - When("Publishing to a queue that exists", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"test"}, + Context("getUrlForQueueName", func() { + When("List queues returns an error", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock).(*SQSQueueService) + + By("Calling ListQueues and receiving an error") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(nil, fmt.Errorf("mock-error")) + + _, err := plugin.getUrlForQueueName("test-queue") + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Encountered an error retrieving the queue list: mock-error")) + ctrl.Finish() }) - plugin := sqs_service.NewWithClient(sqsMock) + }) + + When("No queues exist", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock).(*SQSQueueService) + + By("Calling ListQueues and receiving no queue") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{}, + }, nil) - It("Should publish the message", func() { - _, err := plugin.SendBatch("test", []queue.NitricTask{ + _, err := plugin.getUrlForQueueName("test-queue") + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Unable to find queue with name: test-queue")) + ctrl.Finish() + }) + }) + + When("No queue tags match the nitric name", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock).(*SQSQueueService) + + // Name is in the URL, but that's not important. + queueUrl := aws.String("https://example.com/test-queue") + + By("Calling ListQueues and receiving no queue") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags with the available queues") + sqsMock.EXPECT().ListQueueTags(&sqs.ListQueueTagsInput{QueueUrl: queueUrl}).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + // The nitric name tag doesn't match the expected queue name + "x-nitric-name": aws.String("not-test-queue"), + }, + }, nil) + + By("calling getUrlForQueueName with test-queue") + _, err := plugin.getUrlForQueueName("test-queue") + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Unable to find queue with name: test-queue")) + ctrl.Finish() + }) + }) + }) + + // Tests for the BatchPush method + Context("BatchSend", func() { + When("Sending to a queue that exists", func() { + It("Should send the task to the queue", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) + + queueUrl := aws.String("https://example.com/test-queue") + + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("test-queue"), + }, + }, nil) + + By("Calling SendMessageBatch with the expected batch entries") + sqsMock.EXPECT().SendMessageBatch(&sqs.SendMessageBatchInput{ + QueueUrl: queueUrl, + Entries: []*sqs.SendMessageBatchRequestEntry{ + { + Id: aws.String("1234"), + MessageBody: aws.String(`{"id":"1234","payloadType":"test-payload","payload":{"Test":"Test"}}`), + }, + }, + }).Return(&sqs.SendMessageBatchOutput{}, nil) + + _, err := plugin.SendBatch("test-queue", []queue.NitricTask{ { ID: "1234", PayloadType: "test-payload", @@ -47,139 +143,229 @@ var _ = Describe("Sqs", func() { }, }) + By("Not returning an error") Expect(err).ShouldNot(HaveOccurred()) + ctrl.Finish() }) + }) When("Publishing to a queue that doesn't exist", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{}) - plugin := sqs_service.NewWithClient(sqsMock) + When("List queues returns an error", func() { + It("Should fail to publish the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) - It("Should fail to publish the message", func() { - _, err := plugin.SendBatch("test", []queue.NitricTask{ - { - ID: "1234", - PayloadType: "test-payload", - Payload: map[string]interface{}{ - "Test": "Test", + By("Calling ListQueues and receiving an error") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(nil, fmt.Errorf("mock-error")) + + _, err := plugin.SendBatch("test-queue", []queue.NitricTask{ + { + ID: "1234", + PayloadType: "test-payload", + Payload: map[string]interface{}{ + "Test": "Test", + }, }, - }, - }) + }) - Expect(err).Should(HaveOccurred()) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Encountered an error retrieving the queue list: mock-error")) + ctrl.Finish() + }) }) }) }) - // Tests for the Pop method - Context("Pop", func() { - When("Popping from a queue that exists", func() { + // Tests for the Receive method + Context("Receive", func() { + When("Receive from a queue that exists", func() { When("There is a message on the queue", func() { - mockId := "mockmessageid" - mockReceiptHandle := "mockreceipthandle" - jsonBytes, _ := json.Marshal(events.NitricEvent{ - ID: "mockrequestid", - PayloadType: "mockpayloadtype", - Payload: map[string]interface{}{}, - }) - mockEventJson := string(jsonBytes) + It("Should receive the message", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) + + queueUrl := aws.String("https://example.com/test-queue") - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"mock-queue"}, - Messages: map[string][]*mocks_sqs.Message{ - "mock-queue": { + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("mock-queue"), + }, + }, nil) + + By("Calling ReceiveMessage with the expected inputs") + sqsMock.EXPECT().ReceiveMessage(&sqs.ReceiveMessageInput{ + MaxNumberOfMessages: aws.Int64(int64(10)), + MessageAttributeNames: []*string{ + aws.String(sqs.QueueAttributeNameAll), + }, + QueueUrl: queueUrl, + }).Times(1).Return(&sqs.ReceiveMessageOutput{ + Messages: []*sqs.Message{ { - Id: &mockId, - ReceiptHandle: &mockReceiptHandle, - Body: &mockEventJson, + ReceiptHandle: aws.String("mockreceipthandle"), + Body: aws.String(`{"id":"1234","payloadType":"test-payload","payload":{"Test":"Test"}}`), }, }, - }, - }) - plugin := sqs_service.NewWithClient(sqsMock) + }, nil) - depth := uint32(10) + depth := uint32(10) - It("Should pop the message", func() { - msg, err := plugin.Receive(queue.ReceiveOptions{ + By("Returning the task") + messages, err := plugin.Receive(queue.ReceiveOptions{ QueueName: "mock-queue", Depth: &depth, }) - Expect(msg).To(HaveLen(1)) - + Expect(messages).To(HaveLen(1)) + Expect(messages[0]).To(BeEquivalentTo(queue.NitricTask{ + ID: "1234", + PayloadType: "test-payload", + LeaseID: "mockreceipthandle", + Payload: map[string]interface{}{ + "Test": "Test", + }, + })) Expect(err).ShouldNot(HaveOccurred()) + + ctrl.Finish() }) }) + When("There are no messages on the queue", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"mock-queue"}, - Messages: map[string][]*mocks_sqs.Message{ - // Queue with empty message slice - "mock-queue": make([]*mocks_sqs.Message, 0), - }, - }) - plugin := sqs_service.NewWithClient(sqsMock) - depth := uint32(10) + It("Should receive no messages", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) - It("Should pop the message", func() { - msg, err := plugin.Receive(queue.ReceiveOptions{ + queueUrl := aws.String("https://example.com/test-queue") + + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("mock-queue"), + }, + }, nil) + + By("Calling ReceiveMessage with the expected inputs") + sqsMock.EXPECT().ReceiveMessage(&sqs.ReceiveMessageInput{ + MaxNumberOfMessages: aws.Int64(int64(10)), + MessageAttributeNames: []*string{ + aws.String(sqs.QueueAttributeNameAll), + }, + QueueUrl: queueUrl, + }).Times(1).Return(&sqs.ReceiveMessageOutput{ + Messages: []*sqs.Message{}, + }, nil) + + depth := uint32(10) + + msgs, err := plugin.Receive(queue.ReceiveOptions{ QueueName: "mock-queue", Depth: &depth, }) - Expect(len(msg)).To(Equal(0)) + By("Returning an empty array of tasks") + Expect(msgs).To(HaveLen(0)) + By("Not returning an error") Expect(err).ShouldNot(HaveOccurred()) + + ctrl.Finish() }) }) }) - //When("Popping from a queue that doesn't exist", func() { - When("Popping from a queue that doesn't exist", func() { - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{}, - }) - plugin := sqs_service.NewWithClient(sqsMock) + // Tests for the Complete method + Context("Complete", func() { + When("The message is successfully deleted from SQS", func() { - depth := uint32(10) + // No errors set on mock, 'complete' won't return an error. + It("Should successfully delete the task", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) - It("Should return an error", func() { - _, err := plugin.Receive(queue.ReceiveOptions{ - QueueName: "non-existent-queue", - Depth: &depth, - }) + queueUrl := aws.String("https://example.com/test-queue") - By("Returning an error") - Expect(err).Should(HaveOccurred()) - }) - }) - }) + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) - // Tests for the Complete method - Context("Complete", func() { - When("The message is successfully deleted from SQS", func() { - // No errors set on mock, 'complete' won't return an error. - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - Queues: []string{"test-queue"}, - }) - plugin := sqs_service.NewWithClient(sqsMock) + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("test-queue"), + }, + }, nil) - It("Should not return an error", func() { - err := plugin.Complete("test-queue", "test-id") - Expect(err).ShouldNot(HaveOccurred()) - }) - }) - When("The message fails to delete from SQS", func() { - // No errors set on mock, 'complete' won't return an error. - sqsMock := mocks_sqs.NewMockSqs(&mocks_sqs.MockSqsOptions{ - CompleteError: fmt.Errorf("mock complete error"), + By("Calling SQS with the queue url and task lease id") + sqsMock.EXPECT().DeleteMessage(&sqs.DeleteMessageInput{ + QueueUrl: queueUrl, + ReceiptHandle: aws.String("lease-id"), + }).Times(1).Return( + &sqs.DeleteMessageOutput{}, + nil, + ) + + err := plugin.Complete("test-queue", "lease-id") + + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + + ctrl.Finish() + }) }) - plugin := sqs_service.NewWithClient(sqsMock) - It("Should return an error", func() { - err := plugin.Complete("test-queue", "test-id") - Expect(err).Should(HaveOccurred()) + When("The message fails to delete from SQS", func() { + // No errors set on mock, 'complete' won't return an error. + It("Return an error", func() { + ctrl := gomock.NewController(GinkgoT()) + sqsMock := mocks_sqs.NewMockSQSAPI(ctrl) + plugin := NewWithClient(sqsMock) + + queueUrl := aws.String("http://example.com/queue") + + By("Calling ListQueues to get the queue name") + sqsMock.EXPECT().ListQueues(&sqs.ListQueuesInput{}).Times(1).Return(&sqs.ListQueuesOutput{ + QueueUrls: []*string{queueUrl}, + }, nil) + + By("Calling ListQueueTags to get the x-nitric-name") + sqsMock.EXPECT().ListQueueTags(gomock.Any()).Times(1).Return(&sqs.ListQueueTagsOutput{ + Tags: map[string]*string{ + "x-nitric-name": aws.String("test-queue"), + }, + }, nil) + + By("Calling SQS with the queue url and task lease id") + sqsMock.EXPECT().DeleteMessage(&sqs.DeleteMessageInput{ + QueueUrl: queueUrl, + ReceiptHandle: aws.String("test-id"), + }).Return(nil, fmt.Errorf("mock-error")) + + err := plugin.Complete("test-queue", "test-id") + + By("returning the error") + Expect(err).Should(HaveOccurred()) + + ctrl.Finish() + }) }) }) }) diff --git a/pkg/plugins/queue/task.go b/pkg/plugins/queue/task.go index bd1c53bb8..c27e44b70 100644 --- a/pkg/plugins/queue/task.go +++ b/pkg/plugins/queue/task.go @@ -22,8 +22,8 @@ type FailedTask struct { // NitricTask - A task for asynchronous processing type NitricTask struct { - ID string `json:"id,omitempty"` - LeaseID string `json:"leaseId,omitempty"` - PayloadType string `json:"payloadType,omitempty"` + ID string `json:"id,omitempty" log:"ID"` + LeaseID string `json:"leaseId,omitempty" log:"LeaseID"` + PayloadType string `json:"payloadType,omitempty" log:"PayLoadType"` Payload map[string]interface{} `json:"payload,omitempty"` } diff --git a/pkg/plugins/secret/dev/dev.go b/pkg/plugins/secret/dev/dev.go index cfb82ff98..754aa2cb0 100644 --- a/pkg/plugins/secret/dev/dev.go +++ b/pkg/plugins/secret/dev/dev.go @@ -42,7 +42,12 @@ func (s *DevSecretService) secretFileName(sec *secret.Secret, v string) string { } func (s *DevSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { - newErr := errors.ErrorsWithScope("DevSecretService.Put") + newErr := errors.ErrorsWithScope( + "DevSecretService.Put", + map[string]interface{}{ + "secret": sec, + }, + ) if sec == nil { return nil, newErr(codes.InvalidArgument, "provide non-empty secret", nil) @@ -94,7 +99,12 @@ func (s *DevSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPu } func (s *DevSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { - newErr := errors.ErrorsWithScope("DevSecretService.Access") + newErr := errors.ErrorsWithScope( + "DevSecretService.Access", + map[string]interface{}{ + "version": sv, + }, + ) if sv.Secret.Name == "" { return nil, newErr( diff --git a/pkg/plugins/secret/key_vault/key_vault.go b/pkg/plugins/secret/key_vault/key_vault.go new file mode 100644 index 000000000..7fc275e2f --- /dev/null +++ b/pkg/plugins/secret/key_vault/key_vault.go @@ -0,0 +1,211 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package key_vault_secret_service + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" + "github.com/nitric-dev/membrane/pkg/plugins/secret" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" + "github.com/nitric-dev/membrane/pkg/utils" +) + +type KeyVaultClient interface { + SetSecret(ctx context.Context, vaultBaseURL string, secretName string, parameters keyvault.SecretSetParameters) (result keyvault.SecretBundle, err error) + GetSecret(ctx context.Context, vaultBaseURL string, secretName string, secretVersion string) (result keyvault.SecretBundle, err error) +} + +type KeyVaultSecretService struct { + secret.UnimplementedSecretPlugin + client KeyVaultClient + vaultName string +} + +// versionIdFromUrl - Extracts a secret version ID from a full secret version URL +// the expected versionUrl format is https://{VAULT_NAME}.vault.azure.net/secrets/{SECRET_NAME}/{SECRET_VERSION} +func versionIdFromUrl(versionUrl string) string { + urlParts := strings.Split(versionUrl, "/") + return urlParts[len(urlParts)-1] +} + +func validateNewSecret(sec *secret.Secret, val []byte) error { + if sec == nil { + return fmt.Errorf("provide non-nil secret") + } + if len(sec.Name) == 0 { + return fmt.Errorf("provide non-blank secret name") + } + if len(val) == 0 { + return fmt.Errorf("provide non-blank secret value") + } + + return nil +} + +func validateSecretVersion(sec *secret.SecretVersion) error { + if sec == nil { + return fmt.Errorf("provide non-nil versioned secret") + } + if sec.Secret == nil { + return fmt.Errorf("provide non-nil secret") + } + if len(sec.Secret.Name) == 0 { + return fmt.Errorf("provide non-blank secret name") + } + if len(sec.Version) == 0 { + return fmt.Errorf("provide non-blank secret version") + } + return nil +} + +func (s *KeyVaultSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { + validationErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Put", + map[string]interface{}{ + "secret": "nil", + }, + ) + if err := validateNewSecret(sec, val); err != nil { + return nil, validationErr( + codes.InvalidArgument, + "invalid secret", + err, + ) + } + newErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Put", + map[string]interface{}{ + "secret": sec.Name, + }, + ) + stringVal := string(val[:]) + + result, err := s.client.SetSecret( + context.Background(), + fmt.Sprintf("https://%s.vault.azure.net", s.vaultName), //https://myvault.vault.azure.net. + sec.Name, + keyvault.SecretSetParameters{ + Value: &stringVal, + }, + ) + + if err != nil { + return nil, newErr( + codes.Internal, + "error putting secret", + err, + ) + } + + return &secret.SecretPutResponse{ + SecretVersion: &secret.SecretVersion{ + Secret: &secret.Secret{ + Name: sec.Name, + }, + Version: versionIdFromUrl(*result.ID), + }, + }, nil +} + +func (s *KeyVaultSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { + validationErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Access", + map[string]interface{}{ + "secret-version": "nil", + }, + ) + if err := validateSecretVersion(sv); err != nil { + return nil, validationErr( + codes.Internal, + "invalid secret version", + err, + ) + } + newErr := errors.ErrorsWithScope( + "KeyVaultSecretService.Access", + map[string]interface{}{ + "secret-version": sv.Secret.Name, + }, + ) + + //Key vault will default to latest if an empty string is provided + version := sv.Version + if version == "latest" { + version = "" + } + result, err := s.client.GetSecret( + context.Background(), + fmt.Sprintf("https://%s.vault.azure.net", s.vaultName), //https://myvault.vault.azure.net. + sv.Secret.Name, + version, + ) + if err != nil { + return nil, newErr( + codes.Internal, + "failed to access secret", + err, + ) + } + //Returned Secret ID: https://myvault.vault.azure.net/secrets/mysecret/11a536561da34d6b8b452d880df58f3a + //Split to get the version + return &secret.SecretAccessResponse{ + // Return the original secret version payload + SecretVersion: &secret.SecretVersion{ + Secret: &secret.Secret{ + Name: sv.Secret.Name, + }, + Version: versionIdFromUrl(*result.ID), + }, + Value: []byte(*result.Value), + }, nil +} + +// New - Creates a new Nitric secret service with Azure Key Vault Provider +func New() (secret.SecretService, error) { + vaultName := utils.GetEnv("KVAULT_NAME", "") + if len(vaultName) == 0 { + return nil, fmt.Errorf("KVAULT_NAME not configured") + } + + //Auth requires: + //AZURE_TENANT_ID: Your Azure tenant ID + //AZURE_CLIENT_ID: Your Azure client ID. This will be an app ID from your AAD. + spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.KeyVault) + if err != nil { + return nil, err + } + + client := keyvault.New() + client.Authorizer = autorest.NewBearerAuthorizer(spt) + + return &KeyVaultSecretService{ + client: client, + vaultName: vaultName, + }, nil +} + +func NewWithClient(client KeyVaultClient) secret.SecretService { + return &KeyVaultSecretService{ + client: client, + vaultName: "localvault", + } +} diff --git a/pkg/plugins/secret/key_vault/key_vault_suite_test.go b/pkg/plugins/secret/key_vault/key_vault_suite_test.go new file mode 100644 index 000000000..a8a45252a --- /dev/null +++ b/pkg/plugins/secret/key_vault/key_vault_suite_test.go @@ -0,0 +1,26 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package key_vault_secret_service + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestSecretManager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Key Vault Suite") +} diff --git a/pkg/plugins/secret/key_vault/key_vault_test.go b/pkg/plugins/secret/key_vault/key_vault_test.go new file mode 100644 index 000000000..3a69dca62 --- /dev/null +++ b/pkg/plugins/secret/key_vault/key_vault_test.go @@ -0,0 +1,249 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package key_vault_secret_service + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.1/keyvault" + "github.com/golang/mock/gomock" + mocks "github.com/nitric-dev/membrane/mocks/key_vault" + "github.com/nitric-dev/membrane/pkg/plugins/secret" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Key Vault", func() { + secretName := "secret-name" + secretVersion := "secret-version" + secretVal := []byte("Super Secret Message") + secretID := "https://localvault.vault.azure.net/secret/secret-name/secret-version" + secretString := string(secretVal) + mockSecretResponse := keyvault.SecretBundle{ + ID: &secretID, + Value: &secretString, + } + testSecret := &secret.Secret{ + Name: "secret-name", + } + testSecretVersion := &secret.SecretVersion{ + Secret: testSecret, + Version: secretVersion, + } + + When("Put", func() { + When("Given the Key Vault backend is available", func() { + When("Putting a Secret to an existing secret", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + It("Should successfully store a secret", func() { + // Assert all methods are called at least their number of times + defer ctrl.Finish() + + //Mocking expects + mockSecretClient.EXPECT().SetSecret( + context.Background(), + "https://localvault.vault.azure.net", + testSecret.Name, + gomock.Any(), + ).Return(mockSecretResponse, nil).Times(1) + + response, err := secretPlugin.Put(testSecret, secretVal) + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + By("Returning the service provided version id") + Expect(response.SecretVersion.Version).To(Equal(secretVersion)) + Expect(response.SecretVersion.Secret.Name).To(Equal(secretName)) + }) + }) + + When("Putting a Secret to a non-existent secret", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + It("Should successfully store a secret", func() { + defer ctrl.Finish() + + //Mocking expects + mockSecretClient.EXPECT().SetSecret( + context.Background(), + "https://localvault.vault.azure.net", + testSecret.Name, + gomock.Any(), + ).Return(mockSecretResponse, nil).Times(1) + + response, err := secretPlugin.Put(testSecret, secretVal) + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + By("Returning the correct secret") + Expect(response.SecretVersion.Version).To(Equal(secretVersion)) + Expect(response.SecretVersion.Secret.Name).To(Equal(secretName)) + }) + }) + + When("Putting a nil secret", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should invalidate the secret", func() { + _, err := secretPlugin.Put(nil, secretVal) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) + + When("Putting a secret with an empty name", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should invalidate the secret", func() { + _, err := secretPlugin.Put(&secret.Secret{Name: ""}, secretVal) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) + + When("Putting a secret with an empty value", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should invalidate the secret", func() { + _, err := secretPlugin.Put(testSecret, nil) + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) + }) + }) + + When("Access", func() { + When("Given the Key Vault backend is available", func() { + When("The secret store exists", func() { + When("The secret exists", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should successfully return a secret", func() { + defer ctrl.Finish() + //Mocking expects + mockSecretClient.EXPECT().GetSecret( + context.Background(), + "https://localvault.vault.azure.net", + secretName, + secretVersion, + ).Return(mockSecretResponse, nil).Times(1) + + response, err := secretPlugin.Access(testSecretVersion) + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + By("Returning the correct secret") + Expect(response.Value).To(Equal(secretVal)) + Expect(response.SecretVersion).ToNot(BeNil()) + Expect(response.SecretVersion.Version).To(Equal(secretVersion)) + Expect(response.SecretVersion.Secret.Name).To(Equal(secretName)) + }) + }) + }) + When("The secret doesn't exist", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + It("Should return an error", func() { + defer ctrl.Finish() + + mockSecretClient.EXPECT().GetSecret( + context.Background(), + "https://localvault.vault.azure.net", + secretName, + secretVersion, + ).Return(keyvault.SecretBundle{}, fmt.Errorf("secret does not exist")).Times(1) + + response, err := secretPlugin.Access(testSecretVersion) + By("returning an error") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("secret does not exist")) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty secret version is provided", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should return an error", func() { + defer ctrl.Finish() + + response, err := secretPlugin.Access(nil) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty secret is provided", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should return an error", func() { + defer ctrl.Finish() + + response, err := secretPlugin.Access(&secret.SecretVersion{Secret: nil, Version: secretVersion}) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty secret name is provided", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should return an error", func() { + defer ctrl.Finish() + + response, err := secretPlugin.Access(&secret.SecretVersion{Secret: &secret.Secret{Name: ""}, Version: secretVersion}) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + When("An empty version is provided", func() { + ctrl := gomock.NewController(GinkgoT()) + mockSecretClient := mocks.NewMockKeyVaultClient(ctrl) + secretPlugin := NewWithClient(mockSecretClient) + + It("Should return an error", func() { + defer ctrl.Finish() + + response, err := secretPlugin.Access(&secret.SecretVersion{Secret: testSecret, Version: ""}) + By("returning an error") + Expect(err).Should(HaveOccurred()) + By("returning a nil response") + Expect(response).Should(BeNil()) + }) + }) + }) + }) +}) diff --git a/pkg/plugins/secret/secret_manager/secret_manager.go b/pkg/plugins/secret/secret_manager/secret_manager.go index fe8738924..febe6c53b 100644 --- a/pkg/plugins/secret/secret_manager/secret_manager.go +++ b/pkg/plugins/secret/secret_manager/secret_manager.go @@ -135,7 +135,12 @@ func (s *secretManagerSecretService) ensureSecret(sec *secret.Secret) (*secretma // Put - Creates a new secret if one doesn't exist, or just adds a new secret version func (s *secretManagerSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { - newErr := errors.ErrorsWithScope("SecretManagerSecretService.Put") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Put", + map[string]interface{}{ + "secret": sec, + }, + ) if err := validateNewSecret(sec, val); err != nil { return nil, newErr( @@ -186,7 +191,12 @@ func (s *secretManagerSecretService) Put(sec *secret.Secret, val []byte) (*secre // Get - Retrieves a secret given a name and a version func (s *secretManagerSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { - newErr := errors.ErrorsWithScope("SecretManagerSecretService.Access") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Access", + map[string]interface{}{ + "version": sv, + }, + ) fullName, err := s.buildSecretVersionName(sv) diff --git a/pkg/plugins/secret/secret_manager/secret_manager_test.go b/pkg/plugins/secret/secret_manager/secret_manager_test.go index f2dd00896..5b9cce9f7 100644 --- a/pkg/plugins/secret/secret_manager/secret_manager_test.go +++ b/pkg/plugins/secret/secret_manager/secret_manager_test.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/golang/mock/gomock" - mocks "github.com/nitric-dev/membrane/mocks/mock_secret_manager" + mocks "github.com/nitric-dev/membrane/mocks/secret_manager" "github.com/nitric-dev/membrane/pkg/plugins/secret" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/plugins/secret/secret_suite_test.go b/pkg/plugins/secret/secret_suite_test.go index d2c01c74f..fffd63491 100644 --- a/pkg/plugins/secret/secret_suite_test.go +++ b/pkg/plugins/secret/secret_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package secret +package secret_test import ( "testing" diff --git a/pkg/plugins/secret/secret_test.go b/pkg/plugins/secret/secret_test.go index 510631e4b..7e93178a2 100644 --- a/pkg/plugins/secret/secret_test.go +++ b/pkg/plugins/secret/secret_test.go @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package secret +package secret_test import ( + "github.com/nitric-dev/membrane/pkg/plugins/secret" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Unimplemented Secret Plugin Tests", func() { - uisp := &UnimplementedSecretPlugin{} + uisp := &secret.UnimplementedSecretPlugin{} Context("Put", func() { When("Calling Put on UnimplementedSecretPlugin", func() { diff --git a/pkg/plugins/secret/secrets_manager/secrets_manager.go b/pkg/plugins/secret/secrets_manager/secrets_manager.go index 5745e74fd..bd45394e3 100644 --- a/pkg/plugins/secret/secrets_manager/secrets_manager.go +++ b/pkg/plugins/secret/secrets_manager/secrets_manager.go @@ -49,7 +49,12 @@ func (s *secretsManagerSecretService) validateNewSecret(sec *secret.Secret, val } func (s *secretsManagerSecretService) Put(sec *secret.Secret, val []byte) (*secret.SecretPutResponse, error) { - newErr := errors.ErrorsWithScope("SecretsManagerSecretService.Put") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Put", + map[string]interface{}{ + "secret": sec, + }, + ) if err := s.validateNewSecret(sec, val); err != nil { return nil, newErr( @@ -136,7 +141,12 @@ func (s *secretsManagerSecretService) Put(sec *secret.Secret, val []byte) (*secr } func (s *secretsManagerSecretService) Access(sv *secret.SecretVersion) (*secret.SecretAccessResponse, error) { - newErr := errors.ErrorsWithScope("SecretsManagerSecretService.Access") + newErr := errors.ErrorsWithScope( + "SecretManagerSecretService.Access", + map[string]interface{}{ + "version": sv, + }, + ) if len(sv.Secret.Name) == 0 { return nil, newErr( diff --git a/pkg/plugins/secret/types.go b/pkg/plugins/secret/types.go index ac3f86c13..1d7814e63 100644 --- a/pkg/plugins/secret/types.go +++ b/pkg/plugins/secret/types.go @@ -16,16 +16,16 @@ package secret // Secret - Represents a container for secret versions type Secret struct { - Name string + Name string `log:"Name"` } // SecretVersion - A version of a secret type SecretVersion struct { - Secret *Secret + Secret *Secret `log:"Secret"` // Version - the specific secret version this represents // Specifying "latest" will always retrieve the latest version of the secret - Version string + Version string `log:"Version"` } // SecretAccessResponse - Return value for a secret access request diff --git a/pkg/plugins/storage/azblob/azblob.go b/pkg/plugins/storage/azblob/azblob.go new file mode 100644 index 000000000..849c05d75 --- /dev/null +++ b/pkg/plugins/storage/azblob/azblob.go @@ -0,0 +1,236 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/url" + "time" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/nitric-dev/membrane/pkg/plugins/errors" + "github.com/nitric-dev/membrane/pkg/plugins/errors/codes" + "github.com/nitric-dev/membrane/pkg/plugins/storage" + azblob_service_iface "github.com/nitric-dev/membrane/pkg/plugins/storage/azblob/iface" + azureutils "github.com/nitric-dev/membrane/pkg/providers/azure/utils" + "github.com/nitric-dev/membrane/pkg/utils" +) + +// AzblobStorageService - Nitric membrane storage plugin implementation for Azure Storage +type AzblobStorageService struct { + client azblob_service_iface.AzblobServiceUrlIface + storage.UnimplementedStoragePlugin +} + +func (a *AzblobStorageService) getBlobUrl(bucket string, key string) azblob_service_iface.AzblobBlockBlobUrlIface { + cUrl := a.client.NewContainerURL(bucket) + // Get a new blob for the key name + return cUrl.NewBlockBlobURL(key) +} + +func (a *AzblobStorageService) Read(bucket string, key string) ([]byte, error) { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.Read", + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, + ) + // Get the bucket for this bucket name + blob := a.getBlobUrl(bucket, key) + //// download the blob + r, err := blob.Download( + context.TODO(), + 0, + azblob.CountToEnd, + azblob.BlobAccessConditions{}, + false, + azblob.ClientProvidedKeyOptions{}, + ) + + if err != nil { + return nil, newErr( + codes.Internal, + "Unable to download blob", + err, + ) + } + + // TODO: Configure retries + data := r.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20}) + + return ioutil.ReadAll(data) +} + +func (a *AzblobStorageService) Write(bucket string, key string, object []byte) error { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.Write", + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, + ) + + blob := a.getBlobUrl(bucket, key) + + if _, err := blob.Upload( + context.TODO(), + bytes.NewReader(object), + azblob.BlobHTTPHeaders{}, + azblob.Metadata{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil, + azblob.ClientProvidedKeyOptions{}, + ); err != nil { + return newErr( + codes.Internal, + "Unable to write blob data", + err, + ) + } + + return nil +} + +func (a *AzblobStorageService) Delete(bucket string, key string) error { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.Delete", + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, + ) + + // Get the bucket for this bucket name + blob := a.getBlobUrl(bucket, key) + + if _, err := blob.Delete( + context.TODO(), + azblob.DeleteSnapshotsOptionInclude, + azblob.BlobAccessConditions{}, + ); err != nil { + return newErr( + codes.Internal, + "Unable to delete blob", + err, + ) + } + + return nil +} + +func (s *AzblobStorageService) PreSignUrl(bucket string, key string, operation storage.Operation, expiry uint32) (string, error) { + newErr := errors.ErrorsWithScope( + "AzblobStorageService.PreSignUrl", + map[string]interface{}{ + "bucket": bucket, + "key": key, + "operation": operation.String(), + }, + ) + + blobUrlParts := azblob.NewBlobURLParts(s.getBlobUrl(bucket, key).Url()) + currentTime := time.Now().UTC() + validDuration := currentTime.Add(time.Duration(expiry) * time.Second) + cred, err := s.client.GetUserDelegationCredential(context.TODO(), azblob.NewKeyInfo(currentTime, validDuration), nil, nil) + + if err != nil { + return "", newErr( + codes.Internal, + "could not get user delegation credential", + err, + ) + } + + sigOpts := azblob.BlobSASSignatureValues{ + Protocol: azblob.SASProtocolHTTPS, + ExpiryTime: validDuration, + Permissions: azblob.BlobSASPermissions{ + Read: operation == storage.READ, + Write: operation == storage.WRITE, + }.String(), + BlobName: key, + ContainerName: bucket, + } + + queryParams, err := sigOpts.NewSASQueryParameters(cred) + + if err != nil { + return "", newErr( + codes.Internal, + "error signing query params for URL", + err, + ) + } + + blobUrlParts.SAS = queryParams + url := blobUrlParts.URL() + + return url.String(), nil +} + +const expiryBuffer = 2 * time.Minute + +func tokenRefresherFromSpt(spt *adal.ServicePrincipalToken) azblob.TokenRefresher { + return func(credential azblob.TokenCredential) time.Duration { + if err := spt.Refresh(); err != nil { + fmt.Println("Error refreshing token: ", err) + } else { + tkn := spt.Token() + credential.SetToken(tkn.AccessToken) + + return tkn.Expires().Sub(time.Now().Add(expiryBuffer)) + } + + // Mark the token as already expired + return time.Duration(0) + } +} + +// New - Creates a new instance of the AzblobStorageService +func New() (storage.StorageService, error) { + // TODO: Create a default storage account for the stack??? + // XXX: This will limit a membrane wrapped application + // to accessing a single storage account + blobEndpoint := utils.GetEnv(azureutils.AZURE_STORAGE_BLOB_ENDPOINT, "") + if blobEndpoint == "" { + return nil, fmt.Errorf("failed to determine Azure Storage Blob endpoint, environment variable %s not set", azureutils.AZURE_STORAGE_BLOB_ENDPOINT) + } + + spt, err := azureutils.GetServicePrincipalToken(azure.PublicCloud.ResourceIdentifiers.Storage) + if err != nil { + return nil, err + } + + cTkn := azblob.NewTokenCredential(spt.Token().AccessToken, tokenRefresherFromSpt(spt)) + + var accountURL *url.URL + if accountURL, err = url.Parse(blobEndpoint); err != nil { + return nil, err + } + + pipeline := azblob.NewPipeline(cTkn, azblob.PipelineOptions{}) + client := azblob.NewServiceURL(*accountURL, pipeline) + + return &AzblobStorageService{ + client: azblob_service_iface.AdaptServiceUrl(client), + }, nil +} diff --git a/pkg/plugins/storage/azblob/azblob_suite_test.go b/pkg/plugins/storage/azblob/azblob_suite_test.go new file mode 100644 index 000000000..a1f080c59 --- /dev/null +++ b/pkg/plugins/storage/azblob/azblob_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestAzblob(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Azblob Suite") +} diff --git a/pkg/plugins/storage/azblob/azblob_test.go b/pkg/plugins/storage/azblob/azblob_test.go new file mode 100644 index 000000000..5ce82ae8c --- /dev/null +++ b/pkg/plugins/storage/azblob/azblob_test.go @@ -0,0 +1,345 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/url" + "strings" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + mock_azblob "github.com/nitric-dev/membrane/mocks/azblob" + "github.com/nitric-dev/membrane/pkg/plugins/storage" +) + +var _ = Describe("Azblob", func() { + //Context("New", func() { + // When("", func() { + + // }) + //}) + + Context("Read", func() { + When("Azure returns a successfuly response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + mockDown := mock_azblob.NewMockAzblobDownloadResponse(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully return the read payload", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Download once on the blob with the expected options") + mockBlob.EXPECT().Download( + gomock.Any(), + int64(0), + int64(0), + azblob.BlobAccessConditions{}, + false, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(mockDown, nil) + + By("Reading from the download response") + mockDown.EXPECT().Body(gomock.Any()).Times(1).Return(ioutil.NopCloser(strings.NewReader("file-contents"))) + + data, err := storagePlugin.Read("my-bucket", "my-blob") + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + By("Returning the read data") + Expect(data).To(BeEquivalentTo([]byte("file-contents"))) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should return an error", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Download once on the blob with the expected options") + mockBlob.EXPECT().Download( + gomock.Any(), + int64(0), + int64(0), + azblob.BlobAccessConditions{}, + false, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(nil, fmt.Errorf("Failed to download")) + + _, err := storagePlugin.Read("my-bucket", "my-blob") + + By("Returning an error") + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Context("Write", func() { + When("Azure returns a successful response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully write the blob", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Upload( + gomock.Any(), + bytes.NewReader([]byte("test")), + azblob.BlobHTTPHeaders{}, + azblob.Metadata{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(&azblob.BlockBlobUploadResponse{}, nil) + + err := storagePlugin.Write("my-bucket", "my-blob", []byte("test")) + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should return an error", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Upload( + gomock.Any(), + bytes.NewReader([]byte("test")), + azblob.BlobHTTPHeaders{}, + azblob.Metadata{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil, + azblob.ClientProvidedKeyOptions{}, + ).Times(1).Return(nil, fmt.Errorf("mock-error")) + + err := storagePlugin.Write("my-bucket", "my-blob", []byte("test")) + + By("returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) + + Context("Delete", func() { + When("Azure returns a successful response", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully write the blob", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Delete( + gomock.Any(), + azblob.DeleteSnapshotsOptionInclude, + azblob.BlobAccessConditions{}, + ).Times(1).Return(&azblob.BlobDeleteResponse{}, nil) + + err := storagePlugin.Delete("my-bucket", "my-blob") + + By("Not returning an error") + Expect(err).ToNot(HaveOccurred()) + + crtl.Finish() + }) + }) + + When("Azure returns an error", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should successfully write the blob", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Calling Upload once on the blob with the expected options") + mockBlob.EXPECT().Delete( + gomock.Any(), + azblob.DeleteSnapshotsOptionInclude, + azblob.BlobAccessConditions{}, + ).Times(1).Return(nil, fmt.Errorf("mock-error")) + + err := storagePlugin.Delete("my-bucket", "my-blob") + + By("Not returning an error") + Expect(err).To(HaveOccurred()) + + crtl.Finish() + }) + }) + }) + + Context("PresignUrl", func() { + When("User delegation credentials are accessible", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should return a presigned url", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Retrieving user delegation credentials") + mockAzblob.EXPECT().GetUserDelegationCredential( + context.TODO(), gomock.Any(), gomock.Any(), nil, + ).Return( + azblob.NewUserDelegationCredential("mock-account-name", azblob.UserDelegationKey{}), + nil, + ) + + u, _ := url.Parse("https://fake-account.com/my-bucket/my-blob") + By("Getting the URL") + mockBlob.EXPECT().Url().Return(*u) + + url, err := storagePlugin.PreSignUrl("my-bucket", "my-blob", storage.READ, 3600) + + By("Not returning an error") + Expect(err).ShouldNot(HaveOccurred()) + + By("Returning a pre-signed URL from the computed blob URL") + Expect(url).To(ContainSubstring("https://fake-account.com/my-bucket/my-blob")) + }) + }) + + When("retrieving user delegation credentials fails", func() { + crtl := gomock.NewController(GinkgoT()) + mockAzblob := mock_azblob.NewMockAzblobServiceUrlIface(crtl) + mockContainer := mock_azblob.NewMockAzblobContainerUrlIface(crtl) + mockBlob := mock_azblob.NewMockAzblobBlockBlobUrlIface(crtl) + + storagePlugin := &AzblobStorageService{ + client: mockAzblob, + } + + It("should return an error", func() { + By("Retrieving the Container URL for the requested bucket") + mockAzblob.EXPECT().NewContainerURL("my-bucket").Times(1).Return(mockContainer) + + By("Retrieving the blob url of the requested object") + mockContainer.EXPECT().NewBlockBlobURL("my-blob").Times(1).Return(mockBlob) + + By("Failing to retrieve user delegation credentials") + mockAzblob.EXPECT().GetUserDelegationCredential( + context.TODO(), gomock.Any(), gomock.Any(), nil, + ).Return( + nil, + fmt.Errorf("mock-error"), + ) + + u, _ := url.Parse("https://fake-account.com/my-bucket/my-blob") + By("Getting the URL") + mockBlob.EXPECT().Url().Return(*u) + + url, err := storagePlugin.PreSignUrl("my-bucket", "my-blob", storage.READ, 3600) + + By("Not returning a url") + Expect(url).To(Equal("")) + + By("Returning an error") + Expect(err).Should(HaveOccurred()) + }) + }) + }) +}) diff --git a/pkg/plugins/storage/azblob/iface/adapters.go b/pkg/plugins/storage/azblob/iface/adapters.go new file mode 100644 index 000000000..3eba5451b --- /dev/null +++ b/pkg/plugins/storage/azblob/iface/adapters.go @@ -0,0 +1,69 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service_iface + +import ( + "context" + "io" + "net/url" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +func AdaptServiceUrl(c azblob.ServiceURL) AzblobServiceUrlIface { + return serviceUrl{c} +} + +func AdaptContainerUrl(c azblob.ContainerURL) AzblobContainerUrlIface { + return containerUrl{c} +} + +func AdaptBlobUrl(c azblob.BlockBlobURL) AzblobBlockBlobUrlIface { + return blobUrl{c} +} + +type ( + serviceUrl struct{ c azblob.ServiceURL } + containerUrl struct{ c azblob.ContainerURL } + blobUrl struct{ c azblob.BlockBlobURL } +) + +func (c serviceUrl) NewContainerURL(bucket string) AzblobContainerUrlIface { + return AdaptContainerUrl(c.c.NewContainerURL(bucket)) +} + +func (c serviceUrl) GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.StorageAccountCredential, error) { + return c.c.GetUserDelegationCredential(ctx, info, timeout, requestID) +} + +func (c containerUrl) NewBlockBlobURL(blob string) AzblobBlockBlobUrlIface { + return AdaptBlobUrl(c.c.NewBlockBlobURL(blob)) +} + +func (c blobUrl) Download(ctx context.Context, offset int64, count int64, bac azblob.BlobAccessConditions, f bool, cpk azblob.ClientProvidedKeyOptions) (AzblobDownloadResponse, error) { + return c.c.Download(ctx, offset, count, bac, f, cpk) +} + +func (c blobUrl) Url() url.URL { + return c.c.URL() +} + +func (c blobUrl) Upload(ctx context.Context, r io.ReadSeeker, h azblob.BlobHTTPHeaders, m azblob.Metadata, bac azblob.BlobAccessConditions, att azblob.AccessTierType, btm azblob.BlobTagsMap, cpk azblob.ClientProvidedKeyOptions) (*azblob.BlockBlobUploadResponse, error) { + return c.c.Upload(ctx, r, h, m, bac, att, btm, cpk) +} + +func (c blobUrl) Delete(ctx context.Context, dot azblob.DeleteSnapshotsOptionType, bac azblob.BlobAccessConditions) (*azblob.BlobDeleteResponse, error) { + return c.c.Delete(ctx, dot, bac) +} diff --git a/pkg/plugins/storage/azblob/iface/iface.go b/pkg/plugins/storage/azblob/iface/iface.go new file mode 100644 index 000000000..caced691f --- /dev/null +++ b/pkg/plugins/storage/azblob/iface/iface.go @@ -0,0 +1,51 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azblob_service_iface + +import ( + "context" + "io" + "net/url" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +// AzblobServiceUrlIface - Mockable client interface +// for azblob.ServiceUrl +type AzblobServiceUrlIface interface { + NewContainerURL(string) AzblobContainerUrlIface + GetUserDelegationCredential(ctx context.Context, info azblob.KeyInfo, timeout *int32, requestID *string) (azblob.StorageAccountCredential, error) +} + +// AzblobContainerUrlIface - Mockable client interface +// for azblob.ContainerUrl +type AzblobContainerUrlIface interface { + NewBlockBlobURL(string) AzblobBlockBlobUrlIface +} + +// AzblobBlockBlobUrlIface - Mockable client interface +// for azblob.BlockBlobUrl +type AzblobBlockBlobUrlIface interface { + Url() url.URL + Download(context.Context, int64, int64, azblob.BlobAccessConditions, bool, azblob.ClientProvidedKeyOptions) (AzblobDownloadResponse, error) + Upload(context.Context, io.ReadSeeker, azblob.BlobHTTPHeaders, azblob.Metadata, azblob.BlobAccessConditions, azblob.AccessTierType, azblob.BlobTagsMap, azblob.ClientProvidedKeyOptions) (*azblob.BlockBlobUploadResponse, error) + Delete(context.Context, azblob.DeleteSnapshotsOptionType, azblob.BlobAccessConditions) (*azblob.BlobDeleteResponse, error) +} + +// AzblobDownloadResponse - Mockable client interface +// for azblob.DownloadResponse +type AzblobDownloadResponse interface { + Body(azblob.RetryReaderOptions) io.ReadCloser +} diff --git a/pkg/plugins/storage/boltdb/storage.go b/pkg/plugins/storage/boltdb/storage.go index 7b4b4d63f..34bc22f70 100644 --- a/pkg/plugins/storage/boltdb/storage.go +++ b/pkg/plugins/storage/boltdb/storage.go @@ -15,7 +15,6 @@ package boltdb_storage_service import ( - "fmt" "os" "strings" "time" @@ -45,8 +44,11 @@ type Object struct { func (s *BoltStorageService) Write(bucket string, key string, object []byte) error { newErr := errors.ErrorsWithScope( "BoltStorageService.Write", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + "object.len": len(object), + }, ) if bucket == "" { @@ -109,8 +111,10 @@ func (s *BoltStorageService) Write(bucket string, key string, object []byte) err func (s *BoltStorageService) Read(bucket string, key string) ([]byte, error) { newErr := errors.ErrorsWithScope( "BoltStorageService.Read", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if bucket == "" { @@ -156,8 +160,10 @@ func (s *BoltStorageService) Read(bucket string, key string) ([]byte, error) { func (s *BoltStorageService) Delete(bucket string, key string) error { newErr := errors.ErrorsWithScope( "BoltStorageService.Delete", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if bucket == "" { diff --git a/pkg/plugins/storage/minio/minio.go b/pkg/plugins/storage/minio/minio.go new file mode 100644 index 000000000..af34096f0 --- /dev/null +++ b/pkg/plugins/storage/minio/minio.go @@ -0,0 +1,104 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package minio_storage_service + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/nitric-dev/membrane/pkg/plugins/storage" + s3_service "github.com/nitric-dev/membrane/pkg/plugins/storage/s3" + "github.com/nitric-dev/membrane/pkg/utils" +) + +const ( + MINIO_ENDPOINT_ENV = "MINIO_ENDPOINT" + MINIO_ACCESS_KEY_ENV = "MINIO_ACCESS_KEY" + MINIO_SECRET_KEY_ENV = "MINIO_SECRET_KEY" +) + +type minioConfig struct { + endpoint string + accessKey string + secretKey string +} + +func configFromEnv() (*minioConfig, error) { + endpoint := utils.GetEnv(MINIO_ENDPOINT_ENV, "") + accKey := utils.GetEnv(MINIO_ACCESS_KEY_ENV, "") + secKey := utils.GetEnv(MINIO_SECRET_KEY_ENV, "") + + configErrors := make([]error, 0) + + if endpoint == "" { + configErrors = append(configErrors, fmt.Errorf("%s not configured", MINIO_ENDPOINT_ENV)) + } + + if accKey == "" { + configErrors = append(configErrors, fmt.Errorf("%s not configured", MINIO_ACCESS_KEY_ENV)) + } + + if secKey == "" { + configErrors = append(configErrors, fmt.Errorf("%s not configured", MINIO_SECRET_KEY_ENV)) + } + + if len(configErrors) > 0 { + return nil, fmt.Errorf("configuration errors: %v", configErrors) + } + + return &minioConfig{ + endpoint: endpoint, + accessKey: accKey, + secretKey: secKey, + }, nil +} + +func nameSelector(nitricName string, bucket *s3.Bucket) (bool, error) { + if *bucket.Name == nitricName { + return true, nil + } + + return false, nil +} + +func New() (storage.StorageService, error) { + + conf, err := configFromEnv() + + if err != nil { + return nil, err + } + + // Configure to use MinIO Server + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(conf.accessKey, conf.secretKey, ""), + Endpoint: aws.String(conf.endpoint), + Region: aws.String("us-east-1"), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + newSession, err := session.NewSession(s3Config) + + if err != nil { + return nil, fmt.Errorf("error creating new session") + } + + s3Client := s3.New(newSession) + + return s3_service.NewWithClient(s3Client, s3_service.WithSelector(nameSelector)) +} diff --git a/pkg/plugins/storage/plugin.go b/pkg/plugins/storage/plugin.go index c792f9d68..b1198461c 100644 --- a/pkg/plugins/storage/plugin.go +++ b/pkg/plugins/storage/plugin.go @@ -16,10 +16,23 @@ package storage import "fmt" +type Operation int + +const ( + READ Operation = iota + WRITE = iota +) + +func (op Operation) String() string { + // The order of this array must match the iota order above. + return [2]string{"READ", "WRITE"}[op] +} + type StorageService interface { Read(bucket string, key string) ([]byte, error) Write(bucket string, key string, object []byte) error Delete(bucket string, key string) error + PreSignUrl(bucket string, key string, operation Operation, expiry uint32) (string, error) } type UnimplementedStoragePlugin struct{} @@ -37,3 +50,7 @@ func (*UnimplementedStoragePlugin) Write(bucket string, key string, object []byt func (*UnimplementedStoragePlugin) Delete(bucket string, key string) error { return fmt.Errorf("UNIMPLEMENTED") } + +func (*UnimplementedStoragePlugin) PreSignUrl(bucket string, key string, operation Operation, expiry uint32) (string, error) { + return "", fmt.Errorf("UNIMPLEMENTED") +} diff --git a/pkg/plugins/storage/s3/option.go b/pkg/plugins/storage/s3/option.go new file mode 100644 index 000000000..5054ecbed --- /dev/null +++ b/pkg/plugins/storage/s3/option.go @@ -0,0 +1,33 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3_service + +type S3StorageServiceOption interface { + Apply(*S3StorageService) +} + +type withSelector struct { + selector BucketSelector +} + +func (w *withSelector) Apply(service *S3StorageService) { + service.selector = w.selector +} + +func WithSelector(selector BucketSelector) S3StorageServiceOption { + return &withSelector{ + selector: selector, + } +} diff --git a/pkg/plugins/storage/s3/s3.go b/pkg/plugins/storage/s3/s3.go index ca84dff0e..7d13c87e3 100644 --- a/pkg/plugins/storage/s3/s3.go +++ b/pkg/plugins/storage/s3/s3.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "net/http" + "time" "github.com/nitric-dev/membrane/pkg/utils" @@ -33,14 +34,43 @@ import ( ) const ( - // AWS API neglects to include a constant for this error code. + // ErrCodeNoSuchTagSet - AWS API neglects to include a constant for this error code. ErrCodeNoSuchTagSet = "NoSuchTagSet" ) // S3StorageService - Is the concrete implementation of AWS S3 for the Nitric Storage Plugin type S3StorageService struct { - storage.UnimplementedStoragePlugin - client s3iface.S3API + //storage.UnimplementedStoragePlugin + client s3iface.S3API + selector BucketSelector +} + +type BucketSelector = func(nitricName string, b *s3.Bucket) (bool, error) + +func (s *S3StorageService) tagSelector(name string, bucket *s3.Bucket) (bool, error) { + // TODO: This could be rather slow, it's interesting that they don't return this in the list buckets output + tagout, err := s.client.GetBucketTagging(&s3.GetBucketTaggingInput{ + Bucket: bucket.Name, + }) + + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + // Table not found, try to create and put again + if awsErr.Code() == ErrCodeNoSuchTagSet { + // Ignore buckets with no tags, check the next bucket + return false, nil + } + } + return false, err + } + + for _, tag := range tagout.TagSet { + if *tag.Key == "x-nitric-name" && *tag.Value == name { + return true, nil + } + } + + return false, nil } // getBucketByName - Finds and returns a bucket by it's Nitric name @@ -48,43 +78,41 @@ func (s *S3StorageService) getBucketByName(bucket string) (*s3.Bucket, error) { out, err := s.client.ListBuckets(&s3.ListBucketsInput{}) if err != nil { - return nil, fmt.Errorf("Encountered an error retrieving the bucket list: %v", err) + return nil, fmt.Errorf("encountered an error retrieving the bucket list: %v", err) } for _, b := range out.Buckets { - // TODO: This could be rather slow, it's interesting that they don't return this in the list buckets output - tagout, err := s.client.GetBucketTagging(&s3.GetBucketTaggingInput{ - Bucket: b.Name, - }) + var selected bool = false + var selectErr error = nil - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - // Table not found, try to create and put again - if awsErr.Code() == ErrCodeNoSuchTagSet { - // Ignore buckets with no tags, check the next bucket - continue - } - return nil, err - } + if s.selector == nil { + // if selector is undefined us the default selector + selected, selectErr = s.tagSelector(bucket, b) + } else { + // Use provided selector if one available + selected, selectErr = s.selector(bucket, b) + } + + if selectErr != nil { return nil, err } - for _, tag := range tagout.TagSet { - if *tag.Key == "x-nitric-name" && *tag.Value == bucket { - return b, nil - } + if selected { + return b, nil } } - return nil, fmt.Errorf("Unable to find bucket with name: %s", bucket) + return nil, fmt.Errorf("unable to find bucket with name: %s", bucket) } // Read - Retrieves an item from a bucket func (s *S3StorageService) Read(bucket string, key string) ([]byte, error) { newErr := errors.ErrorsWithScope( "S3StorageService.Read", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if b, err := s.getBucketByName(bucket); err == nil { @@ -117,8 +145,11 @@ func (s *S3StorageService) Read(bucket string, key string) ([]byte, error) { func (s *S3StorageService) Write(bucket string, key string, object []byte) error { newErr := errors.ErrorsWithScope( "S3StorageService.Write", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + "object.len": len(object), + }, ) if b, err := s.getBucketByName(bucket); err == nil { @@ -151,8 +182,10 @@ func (s *S3StorageService) Write(bucket string, key string, object []byte) error func (s *S3StorageService) Delete(bucket string, key string) error { newErr := errors.ErrorsWithScope( "S3StorageService.Delete", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) if b, err := s.getBucketByName(bucket); err == nil { @@ -178,6 +211,60 @@ func (s *S3StorageService) Delete(bucket string, key string) error { return nil } +// PreSignUrl - generates a signed URL which can be used to perform direct operations on a file +// useful for large file uploads/downloads so they can bypass application code and work directly with S3 +func (s *S3StorageService) PreSignUrl(bucket string, key string, operation storage.Operation, expiry uint32) (string, error) { + newErr := errors.ErrorsWithScope( + "S3StorageService.PreSignUrl", + map[string]interface{}{ + "bucket": bucket, + "key": key, + "operation": operation.String(), + }, + ) + + if b, err := s.getBucketByName(bucket); err == nil { + switch operation { + case storage.READ: + req, _ := s.client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: b.Name, + Key: aws.String(key), + }) + url, err := req.Presign(time.Duration(expiry) * time.Second) + if err != nil { + return "", newErr( + codes.Internal, + "failed to generate pre-signed READ URL", + err, + ) + } + return url, err + case storage.WRITE: + req, _ := s.client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: b.Name, + Key: aws.String(key), + }) + url, err := req.Presign(time.Duration(expiry) * time.Second) + if err != nil { + return "", newErr( + codes.Internal, + "failed to generate pre-signed WRITE URL", + err, + ) + } + return url, err + default: + return "", fmt.Errorf("requested operation not supported for pre-signed AWS S3 urls") + } + } else { + return "", newErr( + codes.NotFound, + "unable to locate bucket", + err, + ) + } +} + // New creates a new default S3 storage plugin func New() (storage.StorageService, error) { awsRegion := utils.GetEnv("AWS_REGION", "us-east-1") @@ -188,7 +275,7 @@ func New() (storage.StorageService, error) { }) if sessionError != nil { - return nil, fmt.Errorf("Error creating new AWS session %v", sessionError) + return nil, fmt.Errorf("error creating new AWS session %v", sessionError) } s3Client := s3.New(sess) @@ -199,8 +286,14 @@ func New() (storage.StorageService, error) { } // NewWithClient creates a new S3 Storage plugin and injects the given client -func NewWithClient(client s3iface.S3API) (storage.StorageService, error) { - return &S3StorageService{ +func NewWithClient(client s3iface.S3API, opts ...S3StorageServiceOption) (storage.StorageService, error) { + s3Client := &S3StorageService{ client: client, - }, nil + } + + for _, o := range opts { + o.Apply(s3Client) + } + + return s3Client, nil } diff --git a/pkg/plugins/storage/s3/s3_test.go b/pkg/plugins/storage/s3/s3_test.go index 49872dac5..0d1f111f9 100644 --- a/pkg/plugins/storage/s3/s3_test.go +++ b/pkg/plugins/storage/s3/s3_test.go @@ -15,8 +15,17 @@ package s3_service_test import ( - "github.com/nitric-dev/membrane/pkg/plugins/storage/s3" - "github.com/nitric-dev/membrane/tests/mocks/s3" + "fmt" + "net/http" + "net/url" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/golang/mock/gomock" + mock_s3iface "github.com/nitric-dev/membrane/mocks/s3" + s3_service "github.com/nitric-dev/membrane/pkg/plugins/storage/s3" + mock_s3 "github.com/nitric-dev/membrane/tests/mocks/s3" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -131,4 +140,76 @@ var _ = Describe("S3", func() { }) }) }) + When("PreSignUrl", func() { + When("The bucket exists", func() { + // Set up a mock bucket, with a single item + storage := make(map[string]map[string][]byte) + storage["test-bucket"] = make(map[string][]byte) + crtl := gomock.NewController(GinkgoT()) + mockStorageClient := mock_s3iface.NewMockS3API(crtl) + storagePlugin, _ := s3_service.NewWithClient(mockStorageClient) + + When("A URL is requested for a known operation", func() { + It("Should successfully generate the URL", func() { + + By("Calling ListBuckets to map the bucket name") + mockStorageClient.EXPECT().ListBuckets(gomock.Any()).Times(1).Return(&s3.ListBucketsOutput{ + Buckets: []*s3.Bucket{{ + Name: aws.String("test-bucket-aaa111"), + }}, + }, nil) + + mockStorageClient.EXPECT().GetBucketTagging(gomock.Any()).Times(1).Return(&s3.GetBucketTaggingOutput{TagSet: []*s3.Tag{{ + Key: aws.String("x-nitric-name"), + Value: aws.String("test-bucket"), + }}}, nil) + + presign := 0 + mockStorageClient.EXPECT().PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String("test-bucket-aaa111"), // the real bucket name should be provided here, not the nitric name + Key: aws.String("test-key"), + }).Times(1).Return(&request.Request{ + Operation: &request.Operation{ + Name: "", + HTTPMethod: "", + HTTPPath: "", + Paginator: nil, + // Unfortunately, PutObjectRequest returns a struct, instead of an interface, + // so we can't really mock it. However, if this BeforePresignFn returns an error + // it currently prevents the rest of the presign call and returns a blank url string. + // this is good enough to perform basic testing. + BeforePresignFn: func(r *request.Request) error { + presign += 1 + return fmt.Errorf("test error") + }, + }, + HTTPRequest: &http.Request{Host: "", URL: &url.URL{ + Scheme: "", + Opaque: "", + User: nil, + Host: "aws.example.com", + Path: "", + RawPath: "", + ForceQuery: false, + RawQuery: "", + Fragment: "", + RawFragment: "", + }}, + }, nil) + + url, err := storagePlugin.PreSignUrl("test-bucket", "test-key", 1, uint32(60)) + By("Returning an error") + // We always get an error due to inability to replace the Request with a mock + Expect(err).Should(HaveOccurred()) + + By("Returning a blank url") + // always blank - it's the best we can do without a real mock. + Expect(url).To(Equal("")) + }) + }) + }) + When("The bucket doesn't exist", func() { + + }) + }) }) diff --git a/pkg/plugins/storage/storage/storage.go b/pkg/plugins/storage/storage/storage.go index 986a7792e..ca3c44a34 100644 --- a/pkg/plugins/storage/storage/storage.go +++ b/pkg/plugins/storage/storage/storage.go @@ -31,7 +31,7 @@ import ( ) type StorageStorageService struct { - //storage.UnimplementedStoragePlugin + plugin.UnimplementedStoragePlugin client ifaces_gcloud_storage.StorageClient projectID string } @@ -61,8 +61,10 @@ func (s *StorageStorageService) getBucketByName(bucket string) (ifaces_gcloud_st func (s *StorageStorageService) Read(bucket string, key string) ([]byte, error) { newErr := errors.ErrorsWithScope( "StorageStorageService.Read", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) bucketHandle, err := s.getBucketByName(bucket) @@ -102,8 +104,11 @@ func (s *StorageStorageService) Read(bucket string, key string) ([]byte, error) func (s *StorageStorageService) Write(bucket string, key string, object []byte) error { newErr := errors.ErrorsWithScope( "StorageStorageService.Write", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + "object.len": len(object), + }, ) bucketHandle, err := s.getBucketByName(bucket) @@ -143,8 +148,10 @@ func (s *StorageStorageService) Write(bucket string, key string, object []byte) func (s *StorageStorageService) Delete(bucket string, key string) error { newErr := errors.ErrorsWithScope( "StorageStorageService.Delete", - fmt.Sprintf("bucket=%s", bucket), - fmt.Sprintf("key=%s", key), + map[string]interface{}{ + "bucket": bucket, + "key": key, + }, ) bucketHandle, err := s.getBucketByName(bucket) diff --git a/pkg/providers/azure/README.md b/pkg/providers/azure/README.md index d23f6f062..8ab4640ca 100644 --- a/pkg/providers/azure/README.md +++ b/pkg/providers/azure/README.md @@ -34,4 +34,14 @@ make azure-docker-debian > __Note:__ Separate distributions required between glibc/musl as dynamic linker is used for golang plugin support +### Credentials +AZURE_CLIENT_ID +AZURE_CLIENT_SECRET +AZURE_TENANT_ID + +AZURE_SUBSCRIPTION_ID +AZURE_RESOURCE_GROUP + +#### Key Vault +AZURE_VAULT_NAME diff --git a/pkg/providers/azure/membrane.go b/pkg/providers/azure/membrane.go index e539806bd..6f31fd451 100644 --- a/pkg/providers/azure/membrane.go +++ b/pkg/providers/azure/membrane.go @@ -21,12 +21,14 @@ import ( "os/signal" "syscall" + azqueue_service "github.com/nitric-dev/membrane/pkg/plugins/queue/azqueue" + "github.com/nitric-dev/membrane/pkg/membrane" mongodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/mongodb" - "github.com/nitric-dev/membrane/pkg/plugins/events" + event_grid "github.com/nitric-dev/membrane/pkg/plugins/events/eventgrid" http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" - "github.com/nitric-dev/membrane/pkg/plugins/queue" - "github.com/nitric-dev/membrane/pkg/plugins/storage" + key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" + azblob_service "github.com/nitric-dev/membrane/pkg/plugins/storage/azblob" ) func main() { @@ -39,10 +41,17 @@ func main() { fmt.Println("Failed to load document plugin:", err.Error()) } - eventsPlugin := &events.UnimplementedeventsPlugin{} + eventsPlugin, err := event_grid.New() + if err != nil { + fmt.Println("Failed to load event plugin:", err.Error()) + } gatewayPlugin, _ := http_service.New() - queuePlugin := &queue.UnimplementedQueuePlugin{} - storagePlugin := &storage.UnimplementedStoragePlugin{} + queuePlugin, _ := azqueue_service.New() + storagePlugin, _ := azblob_service.New() + secretPlugin, err := key_vault.New() + if err != nil { + fmt.Println("Failed to load secret plugin:", err.Error()) + } m, err := membrane.New(&membrane.MembraneOptions{ DocumentPlugin: documentPlugin, @@ -50,6 +59,7 @@ func main() { GatewayPlugin: gatewayPlugin, QueuePlugin: queuePlugin, StoragePlugin: storagePlugin, + SecretPlugin: secretPlugin, }) if err != nil { @@ -64,9 +74,9 @@ func main() { select { case membraneError := <-errChan: - fmt.Println(fmt.Sprintf("Membrane Error: %v, exiting", membraneError)) + fmt.Printf("Membrane Error: %v, exiting\n", membraneError) case sigTerm := <-term: - fmt.Println(fmt.Sprintf("Received %v, exiting", sigTerm)) + fmt.Printf("Received %v, exiting\n", sigTerm) } m.Stop() diff --git a/pkg/providers/azure/plugin.go b/pkg/providers/azure/plugin.go index 56d8b016d..74c496880 100644 --- a/pkg/providers/azure/plugin.go +++ b/pkg/providers/azure/plugin.go @@ -18,10 +18,14 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/document" mongodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/mongodb" "github.com/nitric-dev/membrane/pkg/plugins/events" + event_grid "github.com/nitric-dev/membrane/pkg/plugins/events/eventgrid" "github.com/nitric-dev/membrane/pkg/plugins/gateway" http_service "github.com/nitric-dev/membrane/pkg/plugins/gateway/appservice" "github.com/nitric-dev/membrane/pkg/plugins/queue" + "github.com/nitric-dev/membrane/pkg/plugins/secret" + key_vault "github.com/nitric-dev/membrane/pkg/plugins/secret/key_vault" "github.com/nitric-dev/membrane/pkg/plugins/storage" + azblob_service "github.com/nitric-dev/membrane/pkg/plugins/storage/azblob" "github.com/nitric-dev/membrane/pkg/providers" ) @@ -32,6 +36,10 @@ func New() providers.ServiceFactory { return &AzureServiceFactory{} } +func (p *AzureServiceFactory) NewSecretService() (secret.SecretService, error) { + return key_vault.New() +} + // NewDocumentService - Returns a MongoDB based document service func (p *AzureServiceFactory) NewDocumentService() (document.DocumentService, error) { return mongodb_service.New() @@ -39,7 +47,7 @@ func (p *AzureServiceFactory) NewDocumentService() (document.DocumentService, er // NewEventService - Returns Azure _ based events plugin func (p *AzureServiceFactory) NewEventService() (events.EventService, error) { - return &events.UnimplementedeventsPlugin{}, nil + return event_grid.New() } // NewGatewayService - Returns Azure _ Gateway plugin @@ -54,5 +62,5 @@ func (p *AzureServiceFactory) NewQueueService() (queue.QueueService, error) { // NewStorageService - Returns Azure _ based storage plugin func (p *AzureServiceFactory) NewStorageService() (storage.StorageService, error) { - return &storage.UnimplementedStoragePlugin{}, nil + return azblob_service.New() } diff --git a/pkg/providers/azure/utils/auth.go b/pkg/providers/azure/utils/auth.go new file mode 100644 index 000000000..6b5c94325 --- /dev/null +++ b/pkg/providers/azure/utils/auth.go @@ -0,0 +1,50 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure/auth" +) + +// GetServicePrincipalToken - Retrieves the service principal token from env +func GetServicePrincipalToken(resource string) (*adal.ServicePrincipalToken, error) { + config, err := auth.GetSettingsFromEnvironment() + + if err != nil { + return nil, fmt.Errorf("failed to retrieve azure auth settings: %v", err) + } + + if fileCred, fileErr := auth.GetSettingsFromFile(); fileErr == nil { + return fileCred.ServicePrincipalTokenFromClientCredentialsWithResource(resource) + } else if clientCred, clientErr := config.GetClientCredentials(); clientErr == nil { + clientCred.Resource = resource + return clientCred.ServicePrincipalToken() + } else if clientCert, certErr := config.GetClientCertificate(); certErr == nil { + clientCert.Resource = resource + return clientCert.ServicePrincipalToken() + } else if userPass, userErr := config.GetUsernamePassword(); userErr == nil { + userPass.Resource = resource + return userPass.ServicePrincipalToken() + } else { + fmt.Printf("error retrieving credentials:\n -> %v\n -> %v\n -> %v\n -> %v\n", fileErr, clientErr, certErr, userErr) + } + + msiConf := config.GetMSI() + msiConf.Resource = resource + return msiConf.ServicePrincipalToken() +} diff --git a/pkg/providers/azure/utils/const.go b/pkg/providers/azure/utils/const.go new file mode 100644 index 000000000..26f52f8ad --- /dev/null +++ b/pkg/providers/azure/utils/const.go @@ -0,0 +1,21 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +// AZURE_STORAGE_BLOB_ENDPOINT - Endpoint for azblob storage plugin +const AZURE_STORAGE_BLOB_ENDPOINT = "AZURE_STORAGE_ACCOUNT_BLOB_ENDPOINT" + +// AZURE_STORAGE_BLOB_ENDPOINT - Endpoint for azqueue queue plugin +const AZURE_STORAGE_QUEUE_ENDPOINT = "AZURE_STORAGE_ACCOUNT_QUEUE_ENDPOINT" diff --git a/pkg/providers/dev/membrane.go b/pkg/providers/dev/membrane.go index 1bfc3c631..986d7106d 100644 --- a/pkg/providers/dev/membrane.go +++ b/pkg/providers/dev/membrane.go @@ -26,8 +26,8 @@ import ( events_service "github.com/nitric-dev/membrane/pkg/plugins/events/dev" gateway_plugin "github.com/nitric-dev/membrane/pkg/plugins/gateway/dev" queue_service "github.com/nitric-dev/membrane/pkg/plugins/queue/dev" - boltdb_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/boltdb" secret_service "github.com/nitric-dev/membrane/pkg/plugins/secret/dev" + minio_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/minio" ) func main() { @@ -41,7 +41,7 @@ func main() { eventsPlugin, _ := events_service.New() gatewayPlugin, _ := gateway_plugin.New() queuePlugin, _ := queue_service.New() - storagePlugin, _ := boltdb_storage_service.New() + storagePlugin, _ := minio_storage_service.New() m, err := membrane.New(&membrane.MembraneOptions{ DocumentPlugin: documentPlugin, diff --git a/pkg/providers/dev/plugin.go b/pkg/providers/dev/plugin.go index d9b461f14..cd1004b6c 100644 --- a/pkg/providers/dev/plugin.go +++ b/pkg/providers/dev/plugin.go @@ -24,7 +24,7 @@ import ( "github.com/nitric-dev/membrane/pkg/plugins/queue" queue_service "github.com/nitric-dev/membrane/pkg/plugins/queue/dev" "github.com/nitric-dev/membrane/pkg/plugins/storage" - boltdb_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/boltdb" + minio_storage_service "github.com/nitric-dev/membrane/pkg/plugins/storage/minio" "github.com/nitric-dev/membrane/pkg/providers" ) @@ -57,5 +57,5 @@ func (p *DevServiceFactory) NewQueueService() (queue.QueueService, error) { // NewStorageService - Returns local dev storage plugin func (p *DevServiceFactory) NewStorageService() (storage.StorageService, error) { - return boltdb_storage_service.New() + return minio_storage_service.New() } diff --git a/pkg/triggers/http_request.go b/pkg/triggers/http_request.go index 5a8aa09f6..19bfcc03e 100644 --- a/pkg/triggers/http_request.go +++ b/pkg/triggers/http_request.go @@ -22,8 +22,6 @@ import ( // HttpRequest - Storage information that captures a HTTP Request type HttpRequest struct { - // The original Headers - // Header *fasthttp.RequestHeader Header map[string][]string // The original body stream Body []byte @@ -32,7 +30,7 @@ type HttpRequest struct { // The original path Path string // URL query parameters - Query map[string]string + Query map[string][]string } func (*HttpRequest) GetTriggerType() TriggerType { @@ -42,7 +40,7 @@ func (*HttpRequest) GetTriggerType() TriggerType { // FromHttpRequest (constructs a HttpRequest source type from a HttpRequest) func FromHttpRequest(ctx *fasthttp.RequestCtx) *HttpRequest { headerCopy := make(map[string][]string) - queryArgs := make(map[string]string) + queryArgs := make(map[string][]string) ctx.Request.Header.VisitAll(func(key []byte, val []byte) { keyString := string(key) @@ -60,7 +58,13 @@ func FromHttpRequest(ctx *fasthttp.RequestCtx) *HttpRequest { }) ctx.QueryArgs().VisitAll(func(key []byte, val []byte) { - queryArgs[string(key)] = string(val) + k := string(key) + + if queryArgs[k] == nil { + queryArgs[k] = make([]string, 0) + } + + queryArgs[k] = append(queryArgs[k], string(val)) }) return &HttpRequest{ diff --git a/pkg/worker/faas_http_worker.go b/pkg/worker/faas_http_worker.go index 322404ca5..7d06358ec 100644 --- a/pkg/worker/faas_http_worker.go +++ b/pkg/worker/faas_http_worker.go @@ -124,6 +124,13 @@ func (h *FaasHttpWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*trig } } + query := make(map[string]*pb.QueryValue) + for k, v := range trigger.Query { + query[k] = &pb.QueryValue{ + Value: v, + } + } + triggerRequest := &pb.TriggerRequest{ Data: trigger.Body, MimeType: mimeType, @@ -132,7 +139,7 @@ func (h *FaasHttpWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*trig Path: trigger.Path, Headers: headers, Method: trigger.Method, - QueryParams: trigger.Query, + QueryParams: query, }, }, } diff --git a/pkg/worker/faas_worker.go b/pkg/worker/faas_worker.go index b24c5d6f3..a21ac5fe0 100644 --- a/pkg/worker/faas_worker.go +++ b/pkg/worker/faas_worker.go @@ -62,7 +62,7 @@ func (s *FaasWorker) resolveTicket(ID string) (chan *pb.TriggerResponse, error) }() if s.responseQueue[ID] == nil { - return nil, fmt.Errorf("Attempted to resolve ticket that does not exist!") + return nil, fmt.Errorf("attempted to resolve ticket that does not exist!") } return s.responseQueue[ID], nil @@ -94,17 +94,30 @@ func (s *FaasWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*triggers } } + query := make(map[string]*pb.QueryValue) + queryOld := make(map[string]string) + for k, v := range trigger.Query { + if v != nil { + query[k] = &pb.QueryValue{ + Value: v, + } + if len(v) > 0 { + queryOld[k] = v[0] + } + } + } + triggerRequest := &pb.TriggerRequest{ Data: trigger.Body, MimeType: mimeType, Context: &pb.TriggerRequest_Http{ Http: &pb.HttpTriggerContext{ - Path: trigger.Path, - Method: trigger.Method, - QueryParams: trigger.Query, - Headers: headers, - HeadersOld: headersOld, - // TODO: Populate path params + Path: trigger.Path, + Method: trigger.Method, + QueryParams: query, + QueryParamsOld: queryOld, + Headers: headers, + HeadersOld: headersOld, }, }, } @@ -131,7 +144,7 @@ func (s *FaasWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*triggers httpResponse := triggerResponse.GetHttp() if httpResponse == nil { - return nil, fmt.Errorf("Fatal: Error handling event, incorrect response received from function") + return nil, fmt.Errorf("fatal: Error handling event, incorrect response received from function") } fasthttpHeader := &fasthttp.ResponseHeader{} diff --git a/pkg/worker/http_worker.go b/pkg/worker/http_worker.go index a97810b7a..acda63a53 100644 --- a/pkg/worker/http_worker.go +++ b/pkg/worker/http_worker.go @@ -64,7 +64,9 @@ func (h *HttpWorker) HandleHttpRequest(trigger *triggers.HttpRequest) (*triggers httpRequest.SetRequestURI(address) for key, val := range trigger.Query { - httpRequest.URI().QueryArgs().Add(key, val) + for _, v := range val { + httpRequest.URI().QueryArgs().Add(key, v) + } } for key, val := range trigger.Header { diff --git a/tests/mocks/s3/mock.go b/tests/mocks/s3/mock.go index 3d03a48eb..bbbce600e 100644 --- a/tests/mocks/s3/mock.go +++ b/tests/mocks/s3/mock.go @@ -30,7 +30,7 @@ type MockBucket struct { Tags map[string]string } -// MockS3Client - Provides and S3API complient mock interface +// MockS3Client - Provides an S3API compliant mock interface type MockS3Client struct { sync.RWMutex s3iface.S3API diff --git a/tests/mocks/sqs/mocks.go b/tests/mocks/sqs/mocks.go deleted file mode 100644 index d0b14dd3e..000000000 --- a/tests/mocks/sqs/mocks.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2021 Nitric Pty Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mocks_sqs - -import ( - "fmt" - "time" - - "github.com/aws/aws-sdk-go/service/sqs" - "github.com/aws/aws-sdk-go/service/sqs/sqsiface" -) - -type MockSqsOptions struct { - Queues []string - Messages map[string][]*Message - CompleteError error -} - -type Message struct { - Id *string - ReceiptHandle *string - Body *string -} - -type MockSqs struct { - sqsiface.SQSAPI - queues []string - messages map[string][]*Message - completeError error -} - -func (s *MockSqs) ListQueues(in *sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error) { - queueUrls := make([]*string, 0) - - for _, queue := range s.queues { - queueUrls = append(queueUrls, &queue) - } - - return &sqs.ListQueuesOutput{ - QueueUrls: queueUrls, - }, nil -} - -func (s *MockSqs) DeleteMessage(req *sqs.DeleteMessageInput) (*sqs.DeleteMessageOutput, error) { - // If an error has been set on the mock, return it. - if s.completeError != nil { - return nil, s.completeError - } - return &sqs.DeleteMessageOutput{}, nil -} - -func (s *MockSqs) ReceiveMessage(in *sqs.ReceiveMessageInput) (*sqs.ReceiveMessageOutput, error) { - for _, q := range s.queues { - if *in.QueueUrl == q { - mockMessages := s.messages[q] - - if mockMessages == nil || len(mockMessages) < 1 { - return &sqs.ReceiveMessageOutput{}, nil - } - - var messages []*sqs.Message - - for i, m := range mockMessages { - // Only return up to the max number of messages requested. - if int64(i) >= *in.MaxNumberOfMessages { - break - } - messages = append(messages, &sqs.Message{ - Body: m.Body, - ReceiptHandle: m.ReceiptHandle, - }) - mockMessages[i] = nil - } - - res := &sqs.ReceiveMessageOutput{ - Messages: messages, - } - - return res, nil - } - } - - return nil, fmt.Errorf("queue not found") -} - -func (s *MockSqs) SendMessageBatch(in *sqs.SendMessageBatchInput) (*sqs.SendMessageBatchOutput, error) { - for _, q := range s.queues { - if *in.QueueUrl == q { - if s.messages[q] == nil { - s.messages[q] = make([]*Message, 0) - } - - successfulMessages := make([]*sqs.SendMessageBatchResultEntry, 0) - failedTasks := make([]*sqs.BatchResultErrorEntry, 0) - for i, e := range in.Entries { - mockReceiptHandle := fmt.Sprintf("%s%s", string(rune(i)), time.Now()) - - s.messages[q] = append(s.messages[q], &Message{ - Id: e.Id, - Body: e.MessageBody, - ReceiptHandle: &mockReceiptHandle, - }) - - successfulMessages = append(successfulMessages, &sqs.SendMessageBatchResultEntry{ - Id: e.Id, - }) - } - - // TODO: Add a configurable failure mechanism here... - return &sqs.SendMessageBatchOutput{ - Successful: successfulMessages, - Failed: failedTasks, - }, nil - } - } - - return nil, fmt.Errorf("Queue: %s does not exist", *in.QueueUrl) -} - -func NewMockSqs(opts *MockSqsOptions) *MockSqs { - if opts.Messages == nil { - opts.Messages = make(map[string][]*Message) - } - return &MockSqs{ - queues: opts.Queues, - messages: opts.Messages, - completeError: opts.CompleteError, - } -} diff --git a/tests/plugins/docker.go b/tests/plugins/docker.go new file mode 100644 index 000000000..98b1f568f --- /dev/null +++ b/tests/plugins/docker.go @@ -0,0 +1,70 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugins + +import ( + "fmt" + "os" + "os/exec" + "strings" + "syscall" +) + +const shell = "/bin/sh" + +func StartContainer(containerName string, args []string) { + + cmd := exec.Command(shell, "-c", strings.Join(args[:], " ")) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error running %s Image %v : %v \n", containerName, cmd, err) + panic(fmt.Sprintf("Error running %s Image %v : %v", containerName, cmd, err)) + } + + // Makes process killable + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func StopContainer(containerName string) { + // clean up + stopArgs := []string{ + "docker", + "container", + "stop", + containerName, + } + + stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) + + if err := stopCmd.Run(); err != nil { + fmt.Printf("Error stopping %s container %v : %v \n", containerName, stopCmd, err) + panic(fmt.Sprintf("Error stopping Firestore container %v : %v", stopCmd, err)) + } + + removeArgs := []string{ + "docker", + "container", + "rm", + containerName, + } + + removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) + + if err := removeCmd.Run(); err != nil { + fmt.Printf("Error removing %s container %v : %v \n", containerName, removeCmd, err) + panic(fmt.Sprintf("Error removing Firestore container %v : %v", removeCmd, err)) + } +} diff --git a/tests/plugins/document/boltdb/boltdb_test.go b/tests/plugins/document/boltdb/boltdb_test.go index bf8eb2cd1..06a7629d9 100644 --- a/tests/plugins/document/boltdb/boltdb_test.go +++ b/tests/plugins/document/boltdb/boltdb_test.go @@ -48,4 +48,5 @@ var _ = Describe("Bolt", func() { test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) diff --git a/tests/plugins/document/delete.go b/tests/plugins/document/delete.go new file mode 100644 index 000000000..57d8d5725 --- /dev/null +++ b/tests/plugins/document/delete.go @@ -0,0 +1,92 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document_suite + +import ( + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func DeleteTests(docPlugin document.DocumentService) { + Context("Delete", func() { + When("Blank key.Collection.Name", func() { + It("Should return error", func() { + key := document.Key{Id: "1"} + err := docPlugin.Delete(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Blank key.Id", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}} + err := docPlugin.Delete(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Delete", func() { + It("Should delete item successfully", func() { + docPlugin.Set(&UserKey1, UserItem1) + + err := docPlugin.Delete(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&UserKey1) + Expect(doc).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Sub Collection Delete", func() { + It("Should delete item successfully", func() { + docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) + + err := docPlugin.Delete(&Customer1.Orders[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&Customer1.Orders[0].Key) + Expect(doc).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Parent and Sub Collection Delete", func() { + It("Should delete all children", func() { + LoadCustomersData(docPlugin) + + col := document.Collection{ + Name: "orders", + Parent: &document.Key{ + Collection: &document.Collection{ + Name: "customers", + }, + }, + } + + result, err := docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) + Expect(err).To(BeNil()) + Expect(result.Documents).To(HaveLen(5)) + + err = docPlugin.Delete(&Customer1.Key) + Expect(err).ShouldNot(HaveOccurred()) + + err = docPlugin.Delete(&Customer2.Key) + Expect(err).ShouldNot(HaveOccurred()) + + result, err = docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) + Expect(err).To(BeNil()) + Expect(result.Documents).To(HaveLen(0)) + }) + }) + }) +} diff --git a/tests/plugins/document/dynamodb/dynamodb_test.go b/tests/plugins/document/dynamodb/dynamodb_test.go index 2a22ea6db..89cd7b158 100644 --- a/tests/plugins/document/dynamodb/dynamodb_test.go +++ b/tests/plugins/document/dynamodb/dynamodb_test.go @@ -16,11 +16,14 @@ package dynamodb_service_test import ( "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/document/dynamodb" "os" - "os/exec" + "time" + dynamodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/dynamodb" + + "github.com/nitric-dev/membrane/tests/plugins" test "github.com/nitric-dev/membrane/tests/plugins/document" + . "github.com/onsi/ginkgo" "github.com/aws/aws-sdk-go/aws" @@ -28,6 +31,10 @@ import ( "github.com/aws/aws-sdk-go/service/dynamodb" ) +const shell = "/bin/sh" +const containerName = "dynamodb-nitric" +const port = "8000" + var _ = Describe("DynamoDb", func() { defer GinkgoRecover() @@ -36,11 +43,22 @@ var _ = Describe("DynamoDb", func() { os.Setenv("AWS_REGION", "X") // Start Local DynamoDB - dynaCmd := startDynamoProcess() + // Run dynamodb container + args := []string{ + "docker", + "run", + "-d", + "-p " + port + ":" + port, + "--name " + containerName, + "amazon/dynamodb-local:latest", + } + plugins.StartContainer(containerName, args) - // Create DynamoDB client + // // Create DynamoDB client db := createDynamoClient() + testConnection(db) + BeforeEach(func() { // Table names suffixed with 7 alphanumeric chars to match pulumi deployment. createTable(db, "customers-1111111") @@ -57,7 +75,7 @@ var _ = Describe("DynamoDb", func() { }) AfterSuite(func() { - stopDynamoProcess(dynaCmd) + plugins.StopContainer(containerName) }) docPlugin, err := dynamodb_service.NewWithClient(db) @@ -69,40 +87,38 @@ var _ = Describe("DynamoDb", func() { test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) -func startDynamoProcess() *exec.Cmd { - // Start Local DynamoDB - args := []string{ - "-Djava.library.path=/usr/local/dynamodb/DynamoDBLocal_lib", - "-jar", - "/usr/local/dynamodb/DynamoDBLocal.jar", - "-inMemory", - } - cmd := exec.Command("/usr/bin/java", args[:]...) - if err := cmd.Start(); err != nil { - panic(fmt.Sprintf("Error starting Local DynamoDB %v : %v", cmd, err)) - } - fmt.Printf("Started Local DynamoDB (PID %v) and loading data...\n", cmd.Process.Pid) - - return cmd -} - -func stopDynamoProcess(cmd *exec.Cmd) { - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("failed to kill DynamoDB %v : %v \n", cmd.Process.Pid, err) - } -} - func createDynamoClient() *dynamodb.DynamoDB { sess := session.Must(session.NewSession(&aws.Config{ Region: aws.String("x"), - Endpoint: aws.String("http://127.0.0.1:8000"), + Endpoint: aws.String("http://localhost:8000"), })) return dynamodb.New(sess) } +func testConnection(db *dynamodb.DynamoDB) { + input := &dynamodb.ListTablesInput{} + + if _, err := db.ListTables(input); err != nil { + // Wait for Java DynamoDB process to get started + time.Sleep(2 * time.Second) + + if _, err := db.ListTables(input); err != nil { + time.Sleep(4 * time.Second) + + if _, err := db.ListTables(input); err != nil { + fmt.Printf("DynamoDB connection error: %v \n", err) + panic(err) + } + } else { + return + } + } +} + func createTable(db *dynamodb.DynamoDB, tableName string) { input := &dynamodb.CreateTableInput{ AttributeDefinitions: []*dynamodb.AttributeDefinition{ @@ -132,7 +148,7 @@ func createTable(db *dynamodb.DynamoDB, tableName string) { TableName: aws.String(tableName), Tags: []*dynamodb.Tag{ { - Key: aws.String("x-nitric-name"), + Key: aws.String("x-nitric-name"), Value: aws.String(tableName), }, }, @@ -152,4 +168,4 @@ func deleteTable(db *dynamodb.DynamoDB, tableName string) { if err != nil { panic(fmt.Sprintf("Error calling DeleteTable: %s", err)) } -} \ No newline at end of file +} diff --git a/tests/plugins/document/firestore/firestore_test.go b/tests/plugins/document/firestore/firestore_test.go index 90d249069..8904a7f65 100644 --- a/tests/plugins/document/firestore/firestore_test.go +++ b/tests/plugins/document/firestore/firestore_test.go @@ -17,43 +17,19 @@ package firestore_service_test import ( "context" "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/document/firestore" "os" - "os/exec" - "syscall" + + firestore_service "github.com/nitric-dev/membrane/pkg/plugins/document/firestore" "cloud.google.com/go/firestore" + "github.com/nitric-dev/membrane/tests/plugins" test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" ) -func startFirestoreProcess() *exec.Cmd { - // Start Local DynamoDB - os.Setenv("FIRESTORE_EMULATOR_HOST", "localhost:8080") - - // Create Firestore Process - args := []string{ - "beta", - "emulators", - "firestore", - "start", - "--host-port=localhost:8080", - } - cmd := exec.Command("gcloud", args[:]...) - if err := cmd.Start(); err != nil { - panic(fmt.Sprintf("Error starting Firestore Emulator %v : %v", cmd, err)) - } - // Makes process killable - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - - return cmd -} - -func stopFirestoreProcess(cmd *exec.Cmd) { - if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { - fmt.Printf("\nFailed to kill Firestore %v : %v \n", cmd.Process.Pid, err) - } -} +const shell = "/bin/sh" +const containerName = "firestore-nitric" +const port = "8080" func createFirestoreClient(ctx context.Context) *firestore.Client { client, err := firestore.NewClient(ctx, "test") @@ -68,14 +44,26 @@ func createFirestoreClient(ctx context.Context) *firestore.Client { var _ = Describe("Firestore", func() { defer GinkgoRecover() + // Start Local DynamoDB + os.Setenv("FIRESTORE_EMULATOR_HOST", "localhost:"+port) + // Start Firestore Emulator - firestoreCmd := startFirestoreProcess() + args := []string{ + "docker", + "run", + "-d", + "-p " + port + ":" + port, + "--env \"FIRESTORE_PROJECT_ID=dummy-project-id\"", + "--name " + containerName, + "mtlynch/firestore-emulator-docker", + } + plugins.StartContainer(containerName, args) ctx := context.Background() db := createFirestoreClient(ctx) AfterSuite(func() { - stopFirestoreProcess(firestoreCmd) + plugins.StopContainer(containerName) }) docPlugin, err := firestore_service.NewWithClient(db, ctx) @@ -87,4 +75,5 @@ var _ = Describe("Firestore", func() { test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) diff --git a/tests/plugins/document/get.go b/tests/plugins/document/get.go new file mode 100644 index 000000000..d915158d8 --- /dev/null +++ b/tests/plugins/document/get.go @@ -0,0 +1,82 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document_suite + +import ( + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func GetTests(docPlugin document.DocumentService) { + Context("Get", func() { + When("Blank key.Collection.Name", func() { + It("Should return error", func() { + key := document.Key{Id: "1"} + _, err := docPlugin.Get(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Blank key.Id", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}} + _, err := docPlugin.Get(&key) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid Get", func() { + It("Should get item successfully", func() { + docPlugin.Set(&UserKey1, UserItem1) + + doc, err := docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Key).To(Equal(&UserKey1)) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) + }) + }) + When("Valid Sub Collection Get", func() { + It("Should store item successfully", func() { + docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) + + doc, err := docPlugin.Get(&Customer1.Orders[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Key).To(Equal(&Customer1.Orders[0].Key)) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) + }) + }) + When("Document Doesn't Exist", func() { + It("Should return NotFound error", func() { + key := document.Key{Collection: &document.Collection{Name: "items"}, Id: "not-exist"} + doc, err := docPlugin.Get(&key) + Expect(doc).To(BeNil()) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found")) + }) + }) + When("Valid Collection Get when there is a Sub Collection", func() { + It("Should store item successfully", func() { + docPlugin.Set(&Customer1.Key, Customer1.Content) + + doc, err := docPlugin.Get(&Customer1.Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Key).To(Equal(&Customer1.Key)) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Content)) + }) + }) + }) +} diff --git a/tests/plugins/document/mongodb/mongodb_test.go b/tests/plugins/document/mongodb/mongodb_test.go index 94df0725e..1fb51afef 100644 --- a/tests/plugins/document/mongodb/mongodb_test.go +++ b/tests/plugins/document/mongodb/mongodb_test.go @@ -17,73 +17,16 @@ package mongodb_service_test import ( "context" "fmt" - "os" - "os/exec" - "strings" - "syscall" mongodb_service "github.com/nitric-dev/membrane/pkg/plugins/document/mongodb" + "github.com/nitric-dev/membrane/tests/plugins" test "github.com/nitric-dev/membrane/tests/plugins/document" . "github.com/onsi/ginkgo" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) -var shell = "/bin/sh" - -func startMongoImage() *exec.Cmd { - // Run mongodb container - args := []string{ - "docker", - "run", - "-d", - "-p 27017-27019:27017-27019", - "--name mongodb-nitric", - "mongo:4.0", - } - - cmd := exec.Command("/bin/sh", "-c", strings.Join(args[:], " ")) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - panic(fmt.Sprintf("Error running MongoDB Image %v : %v", cmd, err)) - } - - // Makes process killable - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - - return cmd -} - -func stopMongoImage(cmd *exec.Cmd) { - - // clean up - stopArgs := []string{ - "docker", - "container", - "stop", - "mongodb-nitric", - } - - stopCmd := exec.Command(shell, "-c", strings.Join(stopArgs[:], " ")) - - if err := stopCmd.Run(); err != nil { - panic(fmt.Sprintf("Error stopping MongoDB container %v : %v", cmd, err)) - } - - removeArgs := []string{ - "docker", - "container", - "rm", - "mongodb-nitric", - } - - removeCmd := exec.Command(shell, "-c", strings.Join(removeArgs[:], " ")) - - if err := removeCmd.Run(); err != nil { - panic(fmt.Sprintf("Error removing MongoDB container %v : %v", cmd, err)) - } -} +const containerName = "mongodb-nitric" func createMongoClient(ctx context.Context) (*mongo.Client, error) { clientOptions := options.Client().ApplyURI("mongodb://localhost:27017").SetDirect(true) @@ -92,7 +35,7 @@ func createMongoClient(ctx context.Context) (*mongo.Client, error) { if clientError != nil { return nil, fmt.Errorf("mongodb error creating client: %v", clientError) } - + connectError := client.Connect(ctx) if connectError != nil { @@ -100,11 +43,11 @@ func createMongoClient(ctx context.Context) (*mongo.Client, error) { } pingError := client.Ping(ctx, nil) - + if pingError != nil { return nil, fmt.Errorf("mongodb unable to connect: %v", pingError) } - + return client, nil } @@ -112,10 +55,18 @@ var _ = Describe("MongoDB", func() { defer GinkgoRecover() // Start Mongo - mongoCmd := startMongoImage() + args := []string{ + "docker", + "run", + "-d", + "-p 27017-27019:27017-27019", + "--name " + containerName, + "mongo:4.0", + } + plugins.StartContainer(containerName, args) AfterSuite(func() { - stopMongoImage(mongoCmd) + plugins.StopContainer(containerName) }) ctx := context.Background() @@ -134,7 +85,8 @@ var _ = Describe("MongoDB", func() { } test.GetTests(docPlugin) - test.SetTests(docPlugin) + test.SetTests(docPlugin) test.DeleteTests(docPlugin) test.QueryTests(docPlugin) + test.QueryStreamTests(docPlugin) }) diff --git a/tests/plugins/document/query.go b/tests/plugins/document/query.go new file mode 100644 index 000000000..450b5edb9 --- /dev/null +++ b/tests/plugins/document/query.go @@ -0,0 +1,570 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document_suite + +import ( + "fmt" + + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func QueryTests(docPlugin document.DocumentService) { + Context("Query", func() { + When("Invalid - blank key.Collection.Name", func() { + It("Should return an error", func() { + result, err := docPlugin.Query(&document.Collection{}, []document.QueryExpression{}, 0, nil) + Expect(result).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Invalid - nil expressions argument", func() { + It("Should return an error", func() { + result, err := docPlugin.Query(&document.Collection{Name: "users"}, nil, 0, nil) + Expect(result).To(BeNil()) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Empty database", func() { + It("Should return empty list", func() { + result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + Expect(result.PagingToken).To(BeNil()) + }) + }) + When("key: {users}, subcol: '', exp: []", func() { + It("Should return all users", func() { + LoadUsersData(docPlugin) + LoadCustomersData(docPlugin) + + result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Collection.Name).To(Equal("users")) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent).To(BeNil()) + } + }) + }) + When("key: {customers, nil}, subcol: '', exp: []", func() { + It("Should return 2 items", func() { + LoadCustomersData(docPlugin) + + result, err := docPlugin.Query(&CustomersColl, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer1.Content["email"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Key)) + Expect(result.PagingToken).To(BeNil()) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US]", func() { + It("Should return 1 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + } + result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer2.Key)) + Expect(result.PagingToken).To(BeNil()) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US, age > 40]", func() { + It("Should return 0 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + {Operand: "age", Operator: ">", Value: "40"}, + } + result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + }) + }) + When("key: {customers, key1}, subcol: orders", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[2].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*result.Documents[2].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[2].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number == 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number == 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number > 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number > 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number < 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number < 1]", func() { + It("Should return 0 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 5 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(5)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(3)) + + for _, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key {customers, nil}, subcol: orders, exps: [type startsWith scooter]", func() { + It("Should return 2 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[1].Content["testName"])) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[1].Key)) + }) + }) + When("key {customers, key1}, subcol: orders, exps: [type startsWith bike/road]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + result, err := docPlugin.Query(&coll, exps, 0, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(1)) + Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {items, nil}, subcol: '', exp: [], limit: 10", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + coll := document.Collection{ + Name: "items", + } + result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 10, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(10)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&coll, []document.QueryExpression{}, 10, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.PagingToken).To(BeNil()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + }) + }) + When("key: {items, nil}, subcol: '', exps: [letter > D], limit: 4", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + coll := document.Collection{ + Name: "items", + } + exps := []document.QueryExpression{ + {Operand: "letter", Operator: ">", Value: "D"}, + } + result, err := docPlugin.Query(&coll, exps, 4, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + + result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + Expect(result.PagingToken).To(BeEmpty()) + }) + }) + When("key: {parentItems, 1}, subcol: items, exp: [], limit: 10", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + result, err := docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(10)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(2)) + Expect(result.PagingToken).To(BeNil()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + }) + }) + When("key: {parentItems, 1}, subcol: items, exps: [letter > D], limit: 4", func() { + It("Should return have multiple pages", func() { + LoadItemsData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "letter", Operator: ">", Value: "D"}, + } + result, err := docPlugin.Query(&ChildItemsCollection, exps, 4, nil) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + dataMap := make(map[string]string) + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + dataMap[val] = val + } + + result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(4)) + Expect(result.PagingToken).ToNot(BeEmpty()) + + // Ensure values are unique + for i, d := range result.Documents { + Expect(d.Key).ToNot(BeNil()) + val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) + if _, found := dataMap[val]; found { + Expect("matching value").ShouldNot(HaveOccurred()) + } + } + + result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) + Expect(result).ToNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result.Documents).To(HaveLen(0)) + Expect(result.PagingToken).To(BeEmpty()) + }) + }) + }) +} diff --git a/tests/plugins/document/set.go b/tests/plugins/document/set.go new file mode 100644 index 000000000..1ad3f3fec --- /dev/null +++ b/tests/plugins/document/set.go @@ -0,0 +1,99 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document_suite + +import ( + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func SetTests(docPlugin document.DocumentService) { + Context("Set", func() { + When("Blank key.Collection.Name", func() { + It("Should return error", func() { + key := document.Key{Id: "1"} + err := docPlugin.Set(&key, UserItem1) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Blank key.Id", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}} + err := docPlugin.Set(&key, UserItem1) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Nil item map", func() { + It("Should return error", func() { + key := document.Key{Collection: &document.Collection{Name: "users"}, Id: "1"} + err := docPlugin.Set(&key, nil) + Expect(err).Should(HaveOccurred()) + }) + }) + When("Valid New Set", func() { + It("Should store new item successfully", func() { + err := docPlugin.Set(&UserKey1, UserItem1) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) + }) + }) + When("Valid Update Set", func() { + It("Should update existing item successfully", func() { + err := docPlugin.Set(&UserKey1, UserItem1) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) + + err = docPlugin.Set(&UserKey1, UserItem2) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err = docPlugin.Get(&UserKey1) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem2["email"])) + }) + }) + When("Valid Sub Collection Set", func() { + It("Should store item successfully", func() { + err := docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&Customer1.Orders[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) + }) + }) + When("Valid Mutliple Sub Collection Set", func() { + It("Should store item successfully", func() { + err := docPlugin.Set(&Customer1.Reviews[0].Key, Customer1.Reviews[0].Content) + Expect(err).ShouldNot(HaveOccurred()) + + doc, err := docPlugin.Get(&Customer1.Reviews[0].Key) + Expect(err).ShouldNot(HaveOccurred()) + Expect(doc).ToNot(BeNil()) + Expect(doc.Content).To(BeEquivalentTo(Customer1.Reviews[0].Content)) + }) + }) + }) +} diff --git a/tests/plugins/document/stream.go b/tests/plugins/document/stream.go new file mode 100644 index 000000000..2a48d7819 --- /dev/null +++ b/tests/plugins/document/stream.go @@ -0,0 +1,445 @@ +// Copyright 2021 Nitric Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document_suite + +import ( + "io" + + "github.com/nitric-dev/membrane/pkg/plugins/document" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func unwrapIter(iter document.DocumentIterator) []*document.Document { + docs := make([]*document.Document, 0) + for { + d, err := iter() + + if err != nil { + Expect(err).To(Equal(io.EOF)) + break + } + + docs = append(docs, d) + } + + return docs +} + +func QueryStreamTests(docPlugin document.DocumentService) { + Context("QueryStream", func() { + // Validation Tests + When("Invalid - blank key.Collection.Name", func() { + It("Should return an iterator that errors", func() { + iter := docPlugin.QueryStream(&document.Collection{}, []document.QueryExpression{}, 0) + Expect(iter).ToNot(BeNil()) + + _, err := iter() + Expect(err).Should(HaveOccurred()) + Expect(err).ToNot(Equal(io.EOF)) + }) + }) + When("Invalid - nil expressions argument", func() { + It("Should return an iterator that errors", func() { + iter := docPlugin.QueryStream(&document.Collection{Name: "users"}, nil, 0) + Expect(iter).ToNot(BeNil()) + + _, err := iter() + Expect(err).Should(HaveOccurred()) + Expect(err).ToNot(Equal(io.EOF)) + }) + }) + + // Query Tests + When("key: {users}, subcol: '', exp: []", func() { + It("Should return all users", func() { + LoadUsersData(docPlugin) + LoadCustomersData(docPlugin) + + iter := docPlugin.QueryStream(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0) + + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(3)) + + for _, d := range docs { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Collection.Name).To(Equal("users")) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent).To(BeNil()) + } + }) + }) + When("key: {customers, nil}, subcol: '', exp: []", func() { + It("Should return 2 items", func() { + LoadCustomersData(docPlugin) + + iter := docPlugin.QueryStream(&CustomersColl, []document.QueryExpression{}, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["email"]).To(BeEquivalentTo(Customer1.Content["email"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US]", func() { + It("Should return 1 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + } + + iter := docPlugin.QueryStream(&CustomersColl, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, nil}, subcol: '', exp: [country == US, age > 40]", func() { + It("Should return 0 item", func() { + LoadCustomersData(docPlugin) + + exps := []document.QueryExpression{ + {Operand: "country", Operator: "==", Value: "US"}, + {Operand: "age", Operator: ">", Value: "40"}, + } + + iter := docPlugin.QueryStream(&CustomersColl, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(0)) + }) + }) + When("key: {customers, key1}, subcol: orders", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + + iter := docPlugin.QueryStream(&coll, []document.QueryExpression{}, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(3)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[2].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*docs[2].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[2].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number == 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number == 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "==", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number > 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(3)) + + for _, d := range docs { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number > 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number < 1]", func() { + It("Should return 0 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(0)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number < 1]", func() { + It("Should return 0 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(0)) + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 5 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(5)) + + for _, d := range docs { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number >= 1]", func() { + It("Should return 3 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: ">=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(3)) + + for _, d := range docs { + Expect(d.Key).ToNot(BeNil()) + Expect(d.Key.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Name).To(Equal("orders")) + Expect(d.Key.Collection.Parent).ToNot(BeNil()) + Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) + Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) + } + }) + }) + When("key: {customers, nil}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 2 orders", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + }) + }) + When("key: {customers, key1}, subcol: orders, exps: [number <= 1]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "number", Operator: "<=", Value: "1"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key {customers, nil}, subcol: orders, exps: [type startsWith scooter]", func() { + It("Should return 2 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &CustomersKey, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(2)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(docs[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[1].Content["testName"])) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) + Expect(*docs[1].Key).To(BeEquivalentTo(Customer2.Orders[1].Key)) + }) + }) + When("key {customers, key1}, subcol: orders, exps: [type startsWith bike/road]", func() { + It("Should return 1 order", func() { + LoadCustomersData(docPlugin) + + coll := document.Collection{ + Name: "orders", + Parent: &Customer1.Key, + } + exps := []document.QueryExpression{ + {Operand: "type", Operator: "startsWith", Value: "scooter"}, + } + + iter := docPlugin.QueryStream(&coll, exps, 0) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(1)) + Expect(docs[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) + Expect(*docs[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) + Expect(*docs[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) + }) + }) + When("key: {items, nil}, subcol: '', exp: [], limit: 10", func() { + It("Should return limited results", func() { + LoadItemsData(docPlugin) + + coll := document.Collection{ + Name: "items", + } + + iter := docPlugin.QueryStream(&coll, []document.QueryExpression{}, 10) + docs := unwrapIter(iter) + + Expect(docs).To(HaveLen(10)) + }) + }) + }) +} diff --git a/tests/plugins/document/suite.go b/tests/plugins/document/suite.go index 8ce718fbd..83208404e 100644 --- a/tests/plugins/document/suite.go +++ b/tests/plugins/document/suite.go @@ -12,14 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package document +package document_suite import ( - "fmt" - "github.com/nitric-dev/membrane/pkg/plugins/document" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) // Simple 'users' collection test data @@ -169,7 +165,7 @@ var Customer1 = Customer{ }, Content: map[string]interface{}{ "title": "Good review", - "stars": "5", + "stars": "5", }, }, }, @@ -326,760 +322,3 @@ func LoadItemsData(docPlugin document.DocumentService) { } // Unit Test Functions -------------------------------------------------------- - -func GetTests(docPlugin document.DocumentService) { - Context("Get", func() { - When("Blank key.Collection.Name", func() { - It("Should return error", func() { - key := document.Key{Id: "1"} - _, err := docPlugin.Get(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Blank key.Id", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}} - _, err := docPlugin.Get(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Get", func() { - It("Should get item successfully", func() { - docPlugin.Set(&UserKey1, UserItem1) - - doc, err := docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Key).To(Equal(&UserKey1)) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) - }) - }) - When("Valid Sub Collection Get", func() { - It("Should store item successfully", func() { - docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) - - doc, err := docPlugin.Get(&Customer1.Orders[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Key).To(Equal(&Customer1.Orders[0].Key)) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) - }) - }) - When("Document Doesn't Exist", func() { - It("Should return NotFound error", func() { - key := document.Key{Collection: &document.Collection{Name: "items"}, Id: "not-exist"} - doc, err := docPlugin.Get(&key) - Expect(doc).To(BeNil()) - Expect(err).Should(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("not found")) - }) - }) - When("Valid Collection Get when there is a Sub Collection", func() { - It("Should store item successfully", func() { - docPlugin.Set(&Customer1.Key, Customer1.Content) - - doc, err := docPlugin.Get(&Customer1.Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Key).To(Equal(&Customer1.Key)) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Content)) - }) - }) - }) -} - -func SetTests(docPlugin document.DocumentService) { - Context("Set", func() { - When("Blank key.Collection.Name", func() { - It("Should return error", func() { - key := document.Key{Id: "1"} - err := docPlugin.Set(&key, UserItem1) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Blank key.Id", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}} - err := docPlugin.Set(&key, UserItem1) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Nil item map", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}, Id: "1"} - err := docPlugin.Set(&key, nil) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid New Set", func() { - It("Should store new item successfully", func() { - err := docPlugin.Set(&UserKey1, UserItem1) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) - }) - }) - When("Valid Update Set", func() { - It("Should update existing item successfully", func() { - err := docPlugin.Set(&UserKey1, UserItem1) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem1["email"])) - - err = docPlugin.Set(&UserKey1, UserItem2) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err = docPlugin.Get(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content["email"]).To(BeEquivalentTo(UserItem2["email"])) - }) - }) - When("Valid Sub Collection Set", func() { - It("Should store item successfully", func() { - err := docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&Customer1.Orders[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Orders[0].Content)) - }) - }) - When("Valid Mutliple Sub Collection Set", func() { - It("Should store item successfully", func() { - err := docPlugin.Set(&Customer1.Reviews[0].Key, Customer1.Reviews[0].Content) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&Customer1.Reviews[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - Expect(doc).ToNot(BeNil()) - Expect(doc.Content).To(BeEquivalentTo(Customer1.Reviews[0].Content)) - }) - }) - }) -} - -func DeleteTests(docPlugin document.DocumentService) { - Context("Delete", func() { - When("Blank key.Collection.Name", func() { - It("Should return error", func() { - key := document.Key{Id: "1"} - err := docPlugin.Delete(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Blank key.Id", func() { - It("Should return error", func() { - key := document.Key{Collection: &document.Collection{Name: "users"}} - err := docPlugin.Delete(&key) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Delete", func() { - It("Should delete item successfully", func() { - docPlugin.Set(&UserKey1, UserItem1) - - err := docPlugin.Delete(&UserKey1) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&UserKey1) - Expect(doc).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Sub Collection Delete", func() { - It("Should delete item successfully", func() { - docPlugin.Set(&Customer1.Orders[0].Key, Customer1.Orders[0].Content) - - err := docPlugin.Delete(&Customer1.Orders[0].Key) - Expect(err).ShouldNot(HaveOccurred()) - - doc, err := docPlugin.Get(&Customer1.Orders[0].Key) - Expect(doc).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Valid Parent and Sub Collection Delete", func() { - It("Should delete all children", func() { - LoadCustomersData(docPlugin) - - col := document.Collection{ - Name: "orders", - Parent: &document.Key{ - Collection: &document.Collection{ - Name: "customers", - }, - }, - } - - result, err := docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) - Expect(err).To(BeNil()) - Expect(result.Documents).To(HaveLen(5)) - - err = docPlugin.Delete(&Customer1.Key) - Expect(err).ShouldNot(HaveOccurred()) - - err = docPlugin.Delete(&Customer2.Key) - Expect(err).ShouldNot(HaveOccurred()) - - result, err = docPlugin.Query(&col, []document.QueryExpression{}, 0, nil) - Expect(err).To(BeNil()) - Expect(result.Documents).To(HaveLen(0)) - }) - }) - }) -} - -func QueryTests(docPlugin document.DocumentService) { - Context("Query", func() { - When("Invalid - blank key.Collection.Name", func() { - It("Should return an error", func() { - result, err := docPlugin.Query(&document.Collection{}, []document.QueryExpression{}, 0, nil) - Expect(result).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Invalid - nil expressions argument", func() { - It("Should return an error", func() { - result, err := docPlugin.Query(&document.Collection{Name: "users"}, nil, 0, nil) - Expect(result).To(BeNil()) - Expect(err).Should(HaveOccurred()) - }) - }) - When("Empty database", func() { - It("Should return empty list", func() { - result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - Expect(result.PagingToken).To(BeNil()) - }) - }) - When("key: {users}, subcol: '', exp: []", func() { - It("Should return all users", func() { - LoadUsersData(docPlugin) - LoadCustomersData(docPlugin) - - result, err := docPlugin.Query(&document.Collection{Name: "users"}, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Collection.Name).To(Equal("users")) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent).To(BeNil()) - } - }) - }) - When("key: {customers, nil}, subcol: '', exp: []", func() { - It("Should return 2 items", func() { - LoadCustomersData(docPlugin) - - result, err := docPlugin.Query(&CustomersColl, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer1.Content["email"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Key)) - Expect(result.PagingToken).To(BeNil()) - }) - }) - When("key: {customers, nil}, subcol: '', exp: [country == US]", func() { - It("Should return 1 item", func() { - LoadCustomersData(docPlugin) - - exps := []document.QueryExpression{ - {Operand: "country", Operator: "==", Value: "US"}, - } - result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["email"]).To(BeEquivalentTo(Customer2.Content["email"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer2.Key)) - Expect(result.PagingToken).To(BeNil()) - }) - }) - When("key: {customers, nil}, subcol: '', exp: [country == US, age > 40]", func() { - It("Should return 0 item", func() { - LoadCustomersData(docPlugin) - - exps := []document.QueryExpression{ - {Operand: "country", Operator: "==", Value: "US"}, - {Operand: "age", Operator: ">", Value: "40"}, - } - result, err := docPlugin.Query(&CustomersColl, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - }) - }) - When("key: {customers, key1}, subcol: orders", func() { - It("Should return 3 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[2].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(*result.Documents[2].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[2].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number == 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "==", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number == 1]", func() { - It("Should return 1 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "==", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number > 1]", func() { - It("Should return 3 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number > 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[1].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[1].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number < 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number < 1]", func() { - It("Should return 0 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number >= 1]", func() { - It("Should return 5 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(5)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number >= 1]", func() { - It("Should return 3 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: ">=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(3)) - - for _, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - Expect(d.Key.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Name).To(Equal("orders")) - Expect(d.Key.Collection.Parent).ToNot(BeNil()) - Expect(d.Key.Collection.Parent.Id).ToNot(Equal("")) - Expect(d.Key.Collection.Parent.Collection.Name).To(Equal("customers")) - } - }) - }) - When("key: {customers, nil}, subcol: orders, exps: [number <= 1]", func() { - It("Should return 2 orders", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[0].Content["testName"])) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[0].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) - }) - }) - When("key: {customers, key1}, subcol: orders, exps: [number <= 1]", func() { - It("Should return 1 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "number", Operator: "<=", Value: "1"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[0].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[0].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key {customers, nil}, subcol: orders, exps: [type startsWith scooter]", func() { - It("Should return 2 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &CustomersKey, - } - exps := []document.QueryExpression{ - {Operand: "type", Operator: "startsWith", Value: "scooter"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(result.Documents[1].Content["testName"]).To(BeEquivalentTo(Customer2.Orders[1].Content["testName"])) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[1].Key.Collection.Parent).To(BeEquivalentTo(Customer2.Key)) - Expect(*result.Documents[1].Key).To(BeEquivalentTo(Customer2.Orders[1].Key)) - }) - }) - When("key {customers, key1}, subcol: orders, exps: [type startsWith bike/road]", func() { - It("Should return 1 order", func() { - LoadCustomersData(docPlugin) - - coll := document.Collection{ - Name: "orders", - Parent: &Customer1.Key, - } - exps := []document.QueryExpression{ - {Operand: "type", Operator: "startsWith", Value: "scooter"}, - } - result, err := docPlugin.Query(&coll, exps, 0, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(1)) - Expect(result.Documents[0].Content["testName"]).To(BeEquivalentTo(Customer1.Orders[2].Content["testName"])) - Expect(*result.Documents[0].Key).To(BeEquivalentTo(Customer1.Orders[2].Key)) - Expect(*result.Documents[0].Key.Collection.Parent).To(BeEquivalentTo(Customer1.Key)) - }) - }) - When("key: {items, nil}, subcol: '', exp: [], limit: 10", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - coll := document.Collection{ - Name: "items", - } - result, err := docPlugin.Query(&coll, []document.QueryExpression{}, 10, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(10)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&coll, []document.QueryExpression{}, 10, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.PagingToken).To(BeNil()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - }) - }) - When("key: {items, nil}, subcol: '', exps: [letter > D], limit: 4", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - coll := document.Collection{ - Name: "items", - } - exps := []document.QueryExpression{ - {Operand: "letter", Operator: ">", Value: "D"}, - } - result, err := docPlugin.Query(&coll, exps, 4, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - - result, err = docPlugin.Query(&coll, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - Expect(result.PagingToken).To(BeEmpty()) - }) - }) - When("key: {parentItems, 1}, subcol: items, exp: [], limit: 10", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - result, err := docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(10)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&ChildItemsCollection, []document.QueryExpression{}, 10, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(2)) - Expect(result.PagingToken).To(BeNil()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - }) - }) - When("key: {parentItems, 1}, subcol: items, exps: [letter > D], limit: 4", func() { - It("Should return have multiple pages", func() { - LoadItemsData(docPlugin) - - exps := []document.QueryExpression{ - {Operand: "letter", Operator: ">", Value: "D"}, - } - result, err := docPlugin.Query(&ChildItemsCollection, exps, 4, nil) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - dataMap := make(map[string]string) - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - dataMap[val] = val - } - - result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(4)) - Expect(result.PagingToken).ToNot(BeEmpty()) - - // Ensure values are unique - for i, d := range result.Documents { - Expect(d.Key).ToNot(BeNil()) - val := fmt.Sprintf("%v", result.Documents[i].Content["letter"]) - if _, found := dataMap[val]; found { - Expect("matching value").ShouldNot(HaveOccurred()) - } - } - - result, err = docPlugin.Query(&ChildItemsCollection, exps, 4, result.PagingToken) - Expect(result).ToNot(BeNil()) - Expect(err).ShouldNot(HaveOccurred()) - Expect(result.Documents).To(HaveLen(0)) - Expect(result.PagingToken).To(BeEmpty()) - }) - }) - }) -}