From b5b405ea56e58fe02523fc42c9a06df56bb44805 Mon Sep 17 00:00:00 2001 From: Ronald Arias Date: Fri, 31 May 2024 14:58:14 -0500 Subject: [PATCH 1/3] lower the amount of repo keys to query artifacts when timeout --- package.json | 2 + src/client.test.ts | 204 ++++++++++++++++++++++----------------------- src/client.ts | 122 +++++++++++++-------------- yarn.lock | 5 ++ 4 files changed, 164 insertions(+), 169 deletions(-) diff --git a/package.json b/package.json index c389a1d..45cf51e 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "@jupiterone/integration-sdk-core": "^12.8.1", "@jupiterone/integration-sdk-dev-tools": "^12.8.1", "@jupiterone/integration-sdk-testing": "^12.8.1", + "@types/lodash": "^4.17.4", "@types/node": "^18.0.0", "@types/node-fetch": "^2.5.7", "fetch-mock-jest": "^1.5.1", @@ -45,6 +46,7 @@ "dependencies": { "@jupiterone/integration-sdk-http-client": "^12.8.1", "lmdb": "^3.0.8", + "lodash": "^4.17.21", "node-fetch": "^2.7.0", "node-match-path": "^0.4.4" } diff --git a/src/client.test.ts b/src/client.test.ts index 519fa78..0e84f92 100644 --- a/src/client.test.ts +++ b/src/client.test.ts @@ -5,14 +5,8 @@ import { } from '@jupiterone/integration-sdk-core'; import { createMockIntegrationLogger } from '@jupiterone/integration-sdk-testing'; import { APIClient } from './client'; -import { ArtifactoryArtifactRef, ArtifactoryArtifactResponse } from './types'; +import { ArtifactoryArtifactRef } from './types'; import { integrationConfig } from '../test/config'; -import { Response } from 'node-fetch'; - -jest.mock('node-fetch', () => require('fetch-mock-jest').sandbox()); - -const fetchMock = require('node-fetch'); -fetchMock.config.overwriteRoutes = false; function getIntegrationLogger(): IntegrationLogger { return createMockIntegrationLogger(); @@ -27,7 +21,7 @@ describe('iterateRepositoryArtifacts', () => { }); afterEach(() => { - fetchMock.reset(); + jest.restoreAllMocks(); }); const mockArtifact = (name: string): ArtifactoryArtifactRef => ({ @@ -44,69 +38,52 @@ describe('iterateRepositoryArtifacts', () => { }); it('should call iteratee for each artifact returned by page', async () => { - const firstResponse: ArtifactoryArtifactResponse = { - results: [mockArtifact('artifact-1'), mockArtifact('artifact-2')], - }; - fetchMock.post(`${baseUrl}/artifactory/api/search/aql`, firstResponse, { - repeat: 1, - }); - - const secondResponse: ArtifactoryArtifactResponse = { - results: [mockArtifact('artifact-3'), mockArtifact('artifact-4')], - }; - - fetchMock.post(`${baseUrl}/artifactory/api/search/aql`, secondResponse, { - repeat: 1, + const responses = [ + { results: [mockArtifact('artifact-1'), mockArtifact('artifact-2')] }, + { results: [mockArtifact('artifact-3'), mockArtifact('artifact-4')] }, + { results: [] }, // Indirectly tests that it should stop pagination when an empty page is encountered + ]; + const mockRetryableRequest = jest.spyOn(client as any, 'retryableRequest'); + let i = 0; + mockRetryableRequest.mockImplementation(() => { + const response = responses[i]; + i++; + return { + json: () => Promise.resolve(response), + }; }); - // Indirectly tests that it should stop pagination when an empty page is encountered - fetchMock.post( - `${baseUrl}/artifactory/api/search/aql`, - { results: [] }, - { - repeat: 1, - }, - ); - const iteratee = jest.fn(); await client.iterateRepositoryArtifacts( ['test-repo', 'test-repo-2'], iteratee, ); - const calls = fetchMock.calls(); - expect(calls).toHaveLength(3); - expect(calls[0][0]).toEqual(`${baseUrl}/artifactory/api/search/aql`); - expect(calls[0][1]).toMatchObject({ + expect(mockRetryableRequest).toHaveBeenCalledTimes(3); + expect(mockRetryableRequest.mock.calls[0][0]).toEqual( + 'artifactory/api/search/aql', + ); + expect(mockRetryableRequest.mock.calls[0][1]).toMatchObject({ body: 'items.find({"$or": [{"repo":"test-repo"},{"repo":"test-repo-2"}]}).offset(0).limit(1000)', method: 'POST', }); expect(iteratee).toHaveBeenCalledTimes(4); - expect(iteratee.mock.calls[0][0]).toEqual( - expect.objectContaining({ - uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-1`, - }), - ); - expect(iteratee.mock.calls[1][0]).toEqual( - expect.objectContaining({ - uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-2`, - }), - ); - expect(iteratee.mock.calls[2][0]).toEqual( - expect.objectContaining({ - uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-3`, - }), - ); - expect(iteratee.mock.calls[3][0]).toEqual( - expect.objectContaining({ - uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-4`, - }), - ); - expect(fetchMock.done()).toBe(true); + for (let i = 1; i <= 4; i++) { + expect(iteratee.mock.calls[i - 1][0]).toEqual( + expect.objectContaining({ + uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-${i}`, + }), + ); + } }); it('should not call iteratee when there are no files in the repository', async () => { - fetchMock.post(`${baseUrl}/artifactory/api/search/aql`, { results: [] }); + const mockRetryableRequest = jest.spyOn(client as any, 'retryableRequest'); + mockRetryableRequest.mockImplementationOnce(() => { + return { + json: () => Promise.resolve({ results: [] }), + }; + }); const iteratee = jest.fn(); await expect( @@ -114,44 +91,18 @@ describe('iterateRepositoryArtifacts', () => { ).resolves.toBeUndefined(); expect(iteratee).not.toHaveBeenCalled(); - expect(fetchMock.done()).toBe(true); }); - it('retries on recoverable error', async () => { - const response: ArtifactoryArtifactResponse = { - results: [mockArtifact('artifact-1')], - }; - - fetchMock - .post( - `${baseUrl}/artifactory/api/search/aql`, - new Response('', { status: 408 }), - { repeat: 1 }, - ) - .post(`${baseUrl}/artifactory/api/search/aql`, response, { repeat: 1 }) - .post( - `${baseUrl}/artifactory/api/search/aql`, - { results: [] }, - { repeat: 1 }, - ); - - const iteratee = jest.fn(); - await client.iterateRepositoryArtifacts(['test-repo'], iteratee); - - expect(iteratee).toHaveBeenCalledTimes(1); - expect(iteratee.mock.calls[0][0]).toEqual( - expect.objectContaining({ - uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-1`, - }), - ); - expect(fetchMock.done()).toBe(true); - }); - - it('throws on unrecoverable error', async () => { - fetchMock.post( - `${baseUrl}/artifactory/api/search/aql`, - new Response('', { status: 404 }), - ); + it('throws any other error', async () => { + const notFoundError = new IntegrationProviderAPIError({ + endpoint: '', + status: 404, + statusText: 'Not Found', + }); + const mockRetryableRequest = jest.spyOn(client as any, 'retryableRequest'); + mockRetryableRequest.mockImplementationOnce(() => { + throw notFoundError; + }); const iteratee = jest.fn(); await expect( @@ -159,21 +110,45 @@ describe('iterateRepositoryArtifacts', () => { ).rejects.toThrow(IntegrationProviderAPIError); expect(iteratee).toHaveBeenCalledTimes(0); - expect(fetchMock.done()).toBe(true); }); - it('should retry with half limit when timeout error encountered', async () => { + it('should retry with half repo keys when timeout error encountered', async () => { const timeoutError = new Error(); - (timeoutError as any).code = 'ATTEMPT_TIMEOUT'; + (timeoutError as any).code = 'ETIMEDOUT'; const mockRetryableRequest = jest.spyOn(client as any, 'retryableRequest'); mockRetryableRequest .mockImplementationOnce(() => { - throw timeoutError; + return Promise.reject(timeoutError); }) .mockImplementationOnce(() => { return { json: () => - Promise.resolve({ results: [mockArtifact('artifact-1')] }), + Promise.resolve({ + results: [ + mockArtifact('artifact-1'), + mockArtifact('artifact-2'), + mockArtifact('artifact-3'), + mockArtifact('artifact-4'), + ], + }), + }; + }) + .mockImplementationOnce(() => { + return { + json: () => Promise.resolve({ results: [] }), + }; + }) + .mockImplementationOnce(() => { + return { + json: () => + Promise.resolve({ + results: [ + mockArtifact('artifact-5'), + mockArtifact('artifact-6'), + mockArtifact('artifact-7'), + mockArtifact('artifact-8'), + ], + }), }; }) .mockImplementationOnce(() => { @@ -184,38 +159,55 @@ describe('iterateRepositoryArtifacts', () => { const iteratee = jest.fn(); await client.iterateRepositoryArtifacts( - ['test-repo', 'test-repo-2'], + ['test-repo', 'test-repo-2', 'test-repo-3', 'test-repo-4'], iteratee, + 4, // initialChunkSize for tests ); - expect(mockRetryableRequest).toHaveBeenCalledTimes(3); + expect(mockRetryableRequest).toHaveBeenCalledTimes(5); expect(mockRetryableRequest).toHaveBeenNthCalledWith( 1, 'artifactory/api/search/aql', expect.objectContaining({ - body: 'items.find({"$or": [{"repo":"test-repo"},{"repo":"test-repo-2"}]}).offset(0).limit(1000)', + body: 'items.find({"$or": [{"repo":"test-repo"},{"repo":"test-repo-2"},{"repo":"test-repo-3"},{"repo":"test-repo-4"}]}).offset(0).limit(1000)', }), ); expect(mockRetryableRequest).toHaveBeenNthCalledWith( 2, 'artifactory/api/search/aql', expect.objectContaining({ - body: 'items.find({"$or": [{"repo":"test-repo"},{"repo":"test-repo-2"}]}).offset(0).limit(500)', + body: 'items.find({"$or": [{"repo":"test-repo"},{"repo":"test-repo-2"}]}).offset(0).limit(1000)', }), ); expect(mockRetryableRequest).toHaveBeenNthCalledWith( 3, 'artifactory/api/search/aql', expect.objectContaining({ - body: 'items.find({"$or": [{"repo":"test-repo"},{"repo":"test-repo-2"}]}).offset(500).limit(500)', + body: 'items.find({"$or": [{"repo":"test-repo"},{"repo":"test-repo-2"}]}).offset(1000).limit(1000)', }), ); - - expect(iteratee).toHaveBeenCalledTimes(1); - expect(iteratee.mock.calls[0][0]).toEqual( + expect(mockRetryableRequest).toHaveBeenNthCalledWith( + 4, + 'artifactory/api/search/aql', expect.objectContaining({ - uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-1`, + body: 'items.find({"$or": [{"repo":"test-repo-3"},{"repo":"test-repo-4"}]}).offset(0).limit(1000)', }), ); + expect(mockRetryableRequest).toHaveBeenNthCalledWith( + 5, + 'artifactory/api/search/aql', + expect.objectContaining({ + body: 'items.find({"$or": [{"repo":"test-repo-3"},{"repo":"test-repo-4"}]}).offset(1000).limit(1000)', + }), + ); + + expect(iteratee).toHaveBeenCalledTimes(8); + for (let i = 1; i <= 8; i++) { + expect(iteratee.mock.calls[i - 1][0]).toEqual( + expect.objectContaining({ + uri: `${baseUrl}/artifactory/test-repo/path/to/artifact/artifact-${i}`, + }), + ); + } }); }); diff --git a/src/client.ts b/src/client.ts index 397c054..df0733a 100644 --- a/src/client.ts +++ b/src/client.ts @@ -2,7 +2,6 @@ import { IntegrationProviderAuthenticationError, IntegrationValidationError, IntegrationLogger, - IntegrationProviderAPIError, } from '@jupiterone/integration-sdk-core'; import { ArtifactEntity, @@ -27,10 +26,10 @@ import { } from './types'; import { BaseAPIClient } from '@jupiterone/integration-sdk-http-client'; import fetch from 'node-fetch'; +import chunk from 'lodash/chunk'; const MAX_ATTEMPTS = 3; const RETRY_DELAY = 3_000; // 3 seconds to start -const TIMEOUT = 60_000; // 3 min timeout. We need this in case Node hangs with ETIMEDOUT const RETRY_FACTOR = 2; const ARTIFACTS_PAGE_LIMIT = 1000; @@ -58,29 +57,8 @@ export class APIClient extends BaseAPIClient { retryOptions: { maxAttempts: MAX_ATTEMPTS, delay: RETRY_DELAY, - timeout: TIMEOUT, + timeout: 0, // disable timeout handling factor: RETRY_FACTOR, - handleError: (err, context, logger) => { - if ( - err instanceof IntegrationProviderAPIError && - ![408, 429, 500, 502, 503, 504].includes(err.status as number) - ) { - logger.info( - { context, err }, - `Hit an unrecoverable error when attempting fetch. Aborting.`, - ); - context.abort(); - } else { - logger.info( - { - err, - attemptNum: context.attemptNum, - attemptsRemaining: context.attemptsRemaining, - }, - `Hit a possibly recoverable error when attempting fetch. Retrying in a moment.`, - ); - } - }, }, }); @@ -279,56 +257,74 @@ export class APIClient extends BaseAPIClient { public async iterateRepositoryArtifacts( keys: string[], iteratee: ResourceIteratee, - initialLimit = ARTIFACTS_PAGE_LIMIT, + initialChunkSize = 100, ): Promise { - let offset = 0; - let currentLimit = initialLimit; - // 2 ** 3 means we'll try to reduce the page size 3 times before giving up. - // E.g. 1000 / 2 = 500 / 2 = 250 / 2 = 125 - const minLimit = Math.max(initialLimit / 2 ** 3, 1); + let chunkSize = initialChunkSize; + let keyChunks = chunk(keys, chunkSize); + const url = 'artifactory/api/search/aql'; const getQuery = (repoKeys: string[], offset: number) => { const reposQuery = JSON.stringify( repoKeys.map((repoKey) => ({ repo: repoKey })), ); - return `items.find({"$or": ${reposQuery}}).offset(${offset}).limit(${currentLimit})`; + return `items.find({"$or": ${reposQuery}}).offset(${offset}).limit(${ARTIFACTS_PAGE_LIMIT})`; }; - let continuePaginating = false; - do { - try { - const response = await this.retryableRequest(url, { - method: 'POST', - body: getQuery(keys, offset), - bodyType: 'text', - headers: { 'Content-Type': 'text/plain' }, - }); - const data = (await response.json()) as ArtifactoryArtifactResponse; - for (const artifact of data.results || []) { - const uri = this.withBaseUrl( - `artifactory/${artifact.repo}/${artifact.path}/${artifact.name}`, - ); - await iteratee({ - ...artifact, - uri, + + let currentChunk = 0; + while (currentChunk < keyChunks.length) { + let continuePaginating = false; + let offset = 0; + do { + try { + const response = await this.retryableRequest(url, { + method: 'POST', + body: getQuery(keyChunks[currentChunk], offset), + bodyType: 'text', + headers: { 'Content-Type': 'text/plain' }, }); - } - continuePaginating = Boolean(data.results?.length); - offset += currentLimit; - } catch (err) { - if (err.code === 'ATTEMPT_TIMEOUT') { - // We'll stop trying to reduce the page size when we reach 125. Starting from 1000 that gives us 3 retries. - const newLimit = Math.max(Math.floor(currentLimit / 2), minLimit); - if (newLimit === currentLimit) { - // We can't reduce the page size any further, throw error. + const data = (await response.json()) as ArtifactoryArtifactResponse; + for (const artifact of data.results || []) { + const uri = this.withBaseUrl( + `artifactory/${artifact.repo}/${artifact.path}/${artifact.name}`, + ); + await iteratee({ + ...artifact, + uri, + }); + } + continuePaginating = Boolean(data.results?.length); + if (!continuePaginating) { + currentChunk++; + } + offset += ARTIFACTS_PAGE_LIMIT; + } catch (err) { + if ( + ['ECONNRESET', 'ETIMEDOUT'].some( + (code) => err.code === code || err.message.includes(code), + ) + ) { + // Try to reduce the chunk size by half + const newChunkSize = Math.max(Math.floor(chunkSize / 2), 1); + if (chunkSize === newChunkSize) { + // We can't reduce the chunk size any further, so just throw the error + throw err; + } + keyChunks = chunk( + keyChunks.slice(currentChunk).flat(), + newChunkSize, + ); + chunkSize = newChunkSize; + currentChunk = 0; + this.logger.warn( + { chunkSize }, + 'Reducing chunk size and retrying.', + ); + } else { throw err; } - currentLimit = newLimit; - continuePaginating = true; - } else { - throw err; } - } - } while (continuePaginating); + } while (continuePaginating); + } } /** diff --git a/yarn.lock b/yarn.lock index a50eb7b..b04e7b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2054,6 +2054,11 @@ "@types/fined" "*" "@types/node" "*" +"@types/lodash@^4.17.4": + version "4.17.4" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.4.tgz#0303b64958ee070059e3a7184048a55159fe20b7" + integrity sha512-wYCP26ZLxaT3R39kiN2+HcJ4kTd3U1waI/cY7ivWYqFP6pW3ZNpvi6Wd6PHZx7T/t8z0vlkXMg3QYLa7DZ/IJQ== + "@types/node-fetch@^2.5.7": version "2.6.4" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660" From b8f89da4f1947c570a67c71601a1f07ada11cb5e Mon Sep 17 00:00:00 2001 From: Ronald Arias Date: Fri, 31 May 2024 14:59:43 -0500 Subject: [PATCH 2/3] remove unused package --- package.json | 1 - yarn.lock | 83 ++-------------------------------------------------- 2 files changed, 2 insertions(+), 82 deletions(-) diff --git a/package.json b/package.json index 45cf51e..f26fbb1 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "@types/lodash": "^4.17.4", "@types/node": "^18.0.0", "@types/node-fetch": "^2.5.7", - "fetch-mock-jest": "^1.5.1", "type-fest": "^0.16.0" }, "dependencies": { diff --git a/yarn.lock b/yarn.lock index b04e7b8..3b54f39 100644 --- a/yarn.lock +++ b/yarn.lock @@ -521,7 +521,7 @@ resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.9.tgz#71cdb00a1ce3a329ce4cbec3a44f9fef35669730" integrity sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ== -"@babel/core@^7.0.0", "@babel/core@^7.11.6", "@babel/core@^7.12.3": +"@babel/core@^7.11.6", "@babel/core@^7.12.3": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.17.tgz#2f9b0b395985967203514b24ee50f9fd0639c866" integrity sha512-2EENLmhpwplDux5PSsZnSbnSkB3tZ6QTksgO25xwEL7pIDcNOMhF5v/s6RzwjMZzZzw9Ofc30gHv5ChCC8pifQ== @@ -756,7 +756,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.22.5" -"@babel/runtime@^7.0.0", "@babel/runtime@^7.5.5": +"@babel/runtime@^7.5.5": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.15.tgz#38f46494ccf6cf020bd4eed7124b425e83e523b8" integrity sha512-T0O+aa+4w0u06iNmapipJXMV4HoUir03hpx3/YqXXhu9xim3w+dVphjFWl1OH8NbZHw5Lbm9k45drDkgq2VNNA== @@ -3024,11 +3024,6 @@ cookie@0.5.0: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== -core-js@^3.0.0: - version "3.32.2" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.32.2.tgz#172fb5949ef468f93b4be7841af6ab1f21992db7" - integrity sha512-pxXSw1mYZPDGvTQqEc5vgIb83jGQKFGYWY76z4a7weZXUolw3G+OvpZqSRcfYOoOVUQJYEPsWeQK8pKEnUtWxQ== - core-util-is@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -3604,29 +3599,6 @@ fb-watchman@^2.0.0: dependencies: bser "2.1.1" -fetch-mock-jest@^1.5.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" - integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== - dependencies: - fetch-mock "^9.11.0" - -fetch-mock@^9.11.0: - version "9.11.0" - resolved "https://registry.yarnpkg.com/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" - integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== - dependencies: - "@babel/core" "^7.0.0" - "@babel/runtime" "^7.0.0" - core-js "^3.0.0" - debug "^4.1.1" - glob-to-regexp "^0.4.0" - is-subset "^0.1.1" - lodash.isequal "^4.5.0" - path-to-regexp "^2.2.1" - querystring "^0.2.0" - whatwg-url "^6.5.0" - figures@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -3883,11 +3855,6 @@ glob-parent@^6.0.2: dependencies: is-glob "^4.0.3" -glob-to-regexp@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" - integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== - glob@^6.0.1: version "6.0.4" resolved "https://registry.yarnpkg.com/glob/-/glob-6.0.4.tgz#0f08860f6a155127b2fadd4f9ce24b1aab6e4d22" @@ -4308,11 +4275,6 @@ is-stream@^2.0.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== -is-subset@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" - integrity sha512-6Ybun0IkarhmEqxXCNw/C0bna6Zb/TkfUX9UbwJtK6ObwAVCxmAP308WWTHviM/zAqXk05cdhYsUsZeGQh99iw== - is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -4996,11 +4958,6 @@ lodash.get@^4.4.2: resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" integrity sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ== -lodash.isequal@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" - integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== - lodash.memoize@4.x: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" @@ -5011,11 +4968,6 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash.sortby@^4.7.0: - version "4.7.0" - resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" - integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== - lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.5: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" @@ -5705,11 +5657,6 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== -path-to-regexp@^2.2.1: - version "2.4.0" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" - integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== - path-type@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" @@ -5857,11 +5804,6 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== -querystring@^0.2.0: - version "0.2.1" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.1.tgz#40d77615bb09d16902a85c3e38aa8b5ed761c2dd" - integrity sha512-wkvS7mL/JMugcup3/rMitHmd9ecIGd2lhFhK9N3UUQ450h66d1r3Y9nvXzQAW1Lq+wyx61k/1pfKS5KuKiyEbg== - querystringify@^2.1.1: version "2.2.0" resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" @@ -6490,13 +6432,6 @@ tough-cookie@~2.5.0: psl "^1.1.28" punycode "^2.1.1" -tr46@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09" - integrity sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA== - dependencies: - punycode "^2.1.0" - tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -6748,11 +6683,6 @@ webidl-conversions@^3.0.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== -webidl-conversions@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" - integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg== - whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" @@ -6761,15 +6691,6 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -whatwg-url@^6.5.0: - version "6.5.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" - integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== - dependencies: - lodash.sortby "^4.7.0" - tr46 "^1.0.1" - webidl-conversions "^4.0.2" - which-pm-runs@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.1.0.tgz#35ccf7b1a0fce87bd8b92a478c9d045785d3bf35" From 842c39a84666f9bede5b0ba7a6b9ae9f5c0ba560 Mon Sep 17 00:00:00 2001 From: Ronald Arias Date: Fri, 31 May 2024 15:23:27 -0500 Subject: [PATCH 3/3] skip build artifacts 500 error and publish message --- src/client.ts | 47 ++++++++++++++++++++++++++++----------------- src/steps/builds.ts | 11 ++++++++++- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/client.ts b/src/client.ts index df0733a..42c146e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -332,9 +332,8 @@ export class APIClient extends BaseAPIClient { * * @param iteratee receives each resource to produce entities/relationships */ - public async iterateBuilds( - iteratee: ResourceIteratee, - ): Promise { + public async iterateBuilds(iteratee: ResourceIteratee) { + const missingBuildArtifacts: string[] = []; const response = await this.retryableRequest('artifactory/api/build'); const jsonResponse: ArtifactoryBuildResponse = await response.json(); @@ -347,25 +346,37 @@ export class APIClient extends BaseAPIClient { for (const buildUri of buildList) { const name = build.uri.split('/')[1]; const number = buildUri.split('/')[1]; - const artifacts = await this.getBuildArtifacts(name, number); - - if (artifacts.length === 0) { - return; - } + try { + const artifacts = await this.getBuildArtifacts(name, number); + if (artifacts.length === 0) { + continue; + } - const repository = artifacts[0] - .split(this.withBaseUrl('artifactory'))[1] - .split('/')[1]; + const repository = artifacts[0] + .split(this.withBaseUrl('artifactory'))[1] + .split('/')[1]; - await iteratee({ - name, - number, - repository, - artifacts, - uri: this.withBaseUrl(`ui/builds${build.uri}`), - }); + await iteratee({ + name, + number, + repository, + artifacts, + uri: this.withBaseUrl(`ui/builds${build.uri}`), + }); + } catch (err) { + if ( + (err.cause.bodyError as string).includes( + 'Binary provider has no content for', + ) + ) { + missingBuildArtifacts.push(name); + } else { + throw err; + } + } } } + return missingBuildArtifacts; } private async getBuildList(buildRef: ArtifactoryBuildRef): Promise { diff --git a/src/steps/builds.ts b/src/steps/builds.ts index 321304d..9651f31 100644 --- a/src/steps/builds.ts +++ b/src/steps/builds.ts @@ -4,6 +4,7 @@ import { IntegrationStep, createDirectRelationship, RelationshipClass, + IntegrationWarnEventName, } from '@jupiterone/integration-sdk-core'; import { IntegrationConfig, ArtifactoryBuild } from '../types'; @@ -38,7 +39,7 @@ export async function fetchBuilds({ const apiClient = createAPIClient(logger, instance.config); try { - await apiClient.iterateBuilds(async (build) => { + const missingBuilds = await apiClient.iterateBuilds(async (build) => { const buildEntity = createBuildEntity(build); if (!jobState.hasKey(buildEntity._key)) { await jobState.addEntity(buildEntity); @@ -60,6 +61,14 @@ export async function fetchBuilds({ } } }); + if (missingBuilds.length) { + logger.publishWarnEvent({ + name: IntegrationWarnEventName.IncompleteData, + description: `There are missing artifacts for builds: ${missingBuilds.join( + ', ', + )}. Due to error "Binary provider has no content for", more details regarding said error message can be found at https://jfrog.com/help/r/artifactory-what-to-do-when-you-get-a-binary-provider-has-no-content-for-error-message`, + }); + } } catch (error) { if (error.status === 404) { logger.warn('No builds found');