Merge pull request #49 from JupiterOne/ip-improve-query-until-limit-r…

…eached IP SPRINT - improve queryUntilLimitReached
JupiterOne · Nov 10, 2023 · df6d08d · df6d08d
2 parents b5cef1b + 53fae69
commit df6d08d
Show file tree

Hide file tree

Showing 11 changed files with 156 additions and 121 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -72,4 +72,4 @@ jobs:
           echo "//registry.npmjs.org/:_authToken=${NPM_AUTH_TOKEN}" > .npmrc
           yarn
           yarn build
-          npm publish ./dist
+          npm publish ./dist
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -9,15 +9,15 @@
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
-name: "CodeQL"
+name: 'CodeQL'
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ['main']
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "main" ]
-    
+    branches: ['main']
+
 jobs:
   analyze:
     name: Analyze
@@ -30,41 +30,40 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'javascript' ]
+        language: ['javascript']
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        queries: security-extended,security-and-quality
+          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          queries: security-extended,security-and-quality
 
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
+      # - run: |
+      #   echo "Run, Build Application using script"
+      #   ./location_of_script_within_repo/buildscript.sh
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/peril.yml b/.github/workflows/peril.yml
@@ -1,9 +1,9 @@
-name: "Peril"
+name: 'Peril'
 
 on:
   pull_request:
 
-env: 
+env:
   TRANSPONDER_DOCKER_IMAGE: 081157560428.dkr.ecr.us-east-1.amazonaws.com/transponder:1
   SECURITY_SCAN_IMAGE: ghcr.io/jupiterone/security-scan:latest
 
@@ -17,74 +17,74 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: Setup Node
-      uses: actions/setup-node@v1
-      with:
-        node-version: 14.x
-
-    - name: Run build
-      run: yarn install
-
-    - name: Get Variables
-      id: get-vars
-      run: |
-        if [[ "${GITHUB_REF}" == 'ref/head/main' && "${GITHUB_EVENT_NAME}" == 'push' ]];
-        then
-          echo ::set-output name=aws-oidc-role::arn:aws:iam::081157560428:role/github-main-role
-        else
-          echo ::set-output name=aws-oidc-role::arn:aws:iam::081157560428:role/github-pull-request-role
-        fi
-    
-    - name: Configure aws credentials
-      uses: aws-actions/configure-aws-credentials@v1
-      with:
-        role-to-assume: ${{ steps.get-vars.outputs.aws-oidc-role }}
-        role-session-name: pr-role-session
-        aws-region: us-east-1
+      - name: Checkout
+        uses: actions/checkout@v3
 
-    - name: ECR login
-      uses: aws-actions/amazon-ecr-login@v1
-      id: amazon-ecr-login
+      - name: Setup Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
 
-    - name: Login to GHCR
-      uses: docker/login-action@v2
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.PACKAGE_TOKEN }}
+      - name: Run build
+        run: yarn install
 
-    - name: Pull security-scan
-      run: |
-        docker pull $SECURITY_SCAN_IMAGE
-    
-    - name: Run security-scan
-      run: |
-        docker run \
-        --user root \
-        -v /var/run/docker.sock:/var/run/docker.sock \
-        -v `pwd`:`pwd` \
-        -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
-        -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
-        -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
-        -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
-        -e GITHUB_REF_NAME=$GITHUB_REF_NAME \
-        -e GITHUB_RUN_NUMBER=$GITHUB_RUN_NUMBER \
-        -e GITHUB_SERVER_URL=$GITHUB_SERVER_URL \
-        -e GITHUB_RUN_ID=$GITHUB_RUN_ID \
-        -e MODE=ci \
-        -w `pwd` $SECURITY_SCAN_IMAGE
+      - name: Get Variables
+        id: get-vars
+        run: |
+          if [[ "${GITHUB_REF}" == 'ref/head/main' && "${GITHUB_EVENT_NAME}" == 'push' ]];
+          then
+            echo ::set-output name=aws-oidc-role::arn:aws:iam::081157560428:role/github-main-role
+          else
+            echo ::set-output name=aws-oidc-role::arn:aws:iam::081157560428:role/github-pull-request-role
+          fi
 
-    - name: Pull transponder
-      run: |
-        docker pull $TRANSPONDER_DOCKER_IMAGE
+      - name: Configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ steps.get-vars.outputs.aws-oidc-role }}
+          role-session-name: pr-role-session
+          aws-region: us-east-1
 
-    - name: Run transponder
-      run: |
-        docker run --rm -v `pwd`:`pwd` -w `pwd` \
-        -e J1_API_KEY=${{ secrets.J1_API_KEY_TRANSPONDER }} \
-        -e J1_API_DOMAIN=${{ secrets.J1_API_DOMAIN_TRANSPONDER }} \
-        -e J1_ACCOUNT_ID=${{ secrets.J1_ACCOUNT_ID_TRANSPONDER }} \
-        $TRANSPONDER_DOCKER_IMAGE
+      - name: ECR login
+        uses: aws-actions/amazon-ecr-login@v1
+        id: amazon-ecr-login
+
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.PACKAGE_TOKEN }}
+
+      - name: Pull security-scan
+        run: |
+          docker pull $SECURITY_SCAN_IMAGE
+
+      - name: Run security-scan
+        run: |
+          docker run \
+          --user root \
+          -v /var/run/docker.sock:/var/run/docker.sock \
+          -v `pwd`:`pwd` \
+          -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
+          -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
+          -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
+          -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
+          -e GITHUB_REF_NAME=$GITHUB_REF_NAME \
+          -e GITHUB_RUN_NUMBER=$GITHUB_RUN_NUMBER \
+          -e GITHUB_SERVER_URL=$GITHUB_SERVER_URL \
+          -e GITHUB_RUN_ID=$GITHUB_RUN_ID \
+          -e MODE=ci \
+          -w `pwd` $SECURITY_SCAN_IMAGE
+
+      - name: Pull transponder
+        run: |
+          docker pull $TRANSPONDER_DOCKER_IMAGE
+
+      - name: Run transponder
+        run: |
+          docker run --rm -v `pwd`:`pwd` -w `pwd` \
+          -e J1_API_KEY=${{ secrets.J1_API_KEY_TRANSPONDER }} \
+          -e J1_API_DOMAIN=${{ secrets.J1_API_DOMAIN_TRANSPONDER }} \
+          -e J1_ACCOUNT_ID=${{ secrets.J1_ACCOUNT_ID_TRANSPONDER }} \
+          $TRANSPONDER_DOCKER_IMAGE
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,13 @@ and this project adheres to
 
 ## [Unreleased]
 
+## 1.8.0 - 2023-11-09
+
+### Changed
+
+- Remove condition to skip `queryUntilLimitReached` when `filterExpression` is
+  not provided
+
 ## 1.7.2- 2021-01-31
 
 ### Added

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/dynamodb-dao",
-  "version": "1.7.6",
+  "version": "1.8.0",
   "description": "DynamoDB Data Access Object (DAO) helper library",
   "repository": {
     "type": "git",

diff --git a/src/index.test.ts b/src/index.test.ts
@@ -435,15 +435,20 @@ test(`#update should be supported`, async () => {
   expect(result).toEqual(data);
 });
 
-test(`#queryUntilLimitReached should call #query if "filterExpression" not provided`, async () => {
+test(`#queryUntilLimitReached should call #query once if "filterExpression" not provided and the limit is reached`, async () => {
   const keyConditionExpression = 'id = :id';
   const attributeValues = { id: uuid() };
   const index = uuid();
-  const limit = 50;
+  const limit = 5;
 
   jest.spyOn(testDao, 'query').mockResolvedValue({
-    lastKey: '',
-    items: [],
+    lastKey: uuid(),
+    items: Array.from({ length: 10 }, (_, i) => ({
+      id: '' + i,
+      index: i,
+      test: uuid(),
+      description: uuid(),
+    })),
   });
 
   const params = {
@@ -456,9 +461,39 @@ test(`#queryUntilLimitReached should call #query if "filterExpression" not provi
 
   await testDao.queryUntilLimitReached(params);
 
+  expect(testDao.query).toHaveBeenCalledTimes(1);
   expect(testDao.query).toHaveBeenCalledWith(params);
 });
 
+test(`#queryUntilLimitReached should call #query the number of times needed until the limit is reached if "filterExpression" not provided`, async () => {
+  const keyConditionExpression = 'id = :id';
+  const attributeValues = { id: uuid() };
+  const index = uuid();
+  const limit = 5;
+
+  jest.spyOn(testDao, 'query').mockResolvedValue({
+    lastKey: uuid(),
+    items: Array.from({ length: 2 }, (_, i) => ({
+      id: '' + i,
+      index: i,
+      test: uuid(),
+      description: uuid(),
+    })),
+  });
+
+  const params = {
+    // `filterExpression` is intentionally not provided
+    index,
+    limit,
+    attributeValues,
+    keyConditionExpression,
+  };
+
+  await testDao.queryUntilLimitReached(params);
+
+  expect(testDao.query).toHaveBeenCalledTimes(3);
+});
+
 test('#scan should allow consistent reads', async () => {
   jest.spyOn(documentClient, 'scan').mockReturnValue({
     promise: () =>

diff --git a/src/index.ts b/src/index.ts
@@ -392,12 +392,6 @@ export default class DynamoDbDao<DataModel, KeySchema> {
   async queryUntilLimitReached(
     params: QueryInputWithLimit
   ): Promise<QueryResult<DataModel>> {
-    if (!params.filterExpression) {
-      // Since there are no filter expressions, DynamoDB will automatically
-      // fulfill the `limit` property.
-      return this.query(params);
-    }
-
     // create a shallow copy of params since we mutate the top level properties
     params = {
       ...params,

diff --git a/test/delete.test.ts b/test/delete.test.ts
@@ -1,5 +1,5 @@
-import TestContext, { documentClient } from './helpers/TestContext';
 import { v4 as uuid } from 'uuid';
+import TestContext, { documentClient } from './helpers/TestContext';
 
 let context: TestContext;
 
@@ -78,7 +78,7 @@ test('should allow for a condition expression to be provided', async () => {
       attributeValues: {
         ':testValue': item.test,
       },
-    },
+    }
   );
 
   return expect(promise).rejects.toThrow('The conditional request failed');
@@ -111,7 +111,7 @@ test('should allow for a expression attribute names to be provided', async () =>
       attributeValues: {
         ':testValue': item.test,
       },
-    },
+    }
   );
 
   return expect(promise).rejects.toThrow('The conditional request failed');