Merge pull request #999 from JupiterOne/clean-errors

upload data improvements
JupiterOne · Dec 14, 2023 · 1630e15 · 1630e15
2 parents 862df33 + 268e788
commit 1630e15
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 12 deletions.
diff --git a/packages/integration-sdk-runtime/src/synchronization/__tests__/index.test.ts b/packages/integration-sdk-runtime/src/synchronization/__tests__/index.test.ts
@@ -578,6 +578,52 @@ describe('uploadDataChunk', () => {
     expect(uploadDataChunkErr).not.toBe(undefined);
     expect(postSpy).toHaveBeenCalledTimes(1);
   });
+
+  it('should clean errors before throwing', async () => {
+    const context = createTestContext();
+    const job = generateSynchronizationJob();
+
+    const type = 'entities';
+    const batch = [];
+
+    const mockLogger = {
+      trace: jest.fn(),
+      debug: jest.fn(),
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn(),
+      fatal: jest.fn(),
+    };
+
+    jest.spyOn(context.apiClient, 'post').mockImplementation(() => {
+      const err = new Error('thing went bad');
+      Object.assign(err, {
+        config: {
+          data: 'Stuff',
+          headers: {
+            Authroization: 'some fake token',
+            'content-type': 'application/json',
+          },
+        },
+      });
+      throw err;
+    });
+
+    await expect(
+      uploadDataChunk({
+        logger: mockLogger as any,
+        apiClient: context.apiClient,
+        jobId: job.id,
+        type,
+        batch,
+      }),
+    ).rejects.toBeInstanceOf(Error);
+    const firstInfoCall = mockLogger.info.mock.calls[0];
+    const args = firstInfoCall[0];
+    const axiosError = args['err'];
+    expect(axiosError.config.data).toBeUndefined();
+    expect(axiosError.config.headers.Authorization).toBeUndefined();
+  });
 });
 
 function createTestContext(
@@ -586,6 +632,7 @@ function createTestContext(
   const apiClient = createApiClient({
     apiBaseUrl: getApiBaseUrl(),
     account: 'test-account',
+    accessToken: 'fake-token',
   });
 
   const logger = createIntegrationLogger({

diff --git a/packages/integration-sdk-runtime/src/synchronization/index.ts b/packages/integration-sdk-runtime/src/synchronization/index.ts
@@ -9,6 +9,8 @@ import {
   SynchronizationJob,
 } from '@jupiterone/integration-sdk-core';
 
+import { AxiosError } from 'axios';
+
 import { IntegrationLogger } from '../logger';
 
 import { ExecuteIntegrationResult } from '../execution';
@@ -467,19 +469,25 @@ export async function uploadDataChunk<
         'Uploading data...',
       );
       const startTime = Date.now();
-      await apiClient.post(
-        `/persister/synchronization/jobs/${jobId}/${type as string}`,
-        {
-          [type]: batch,
-        },
-        {
-          headers: {
-            // NOTE: Other headers that were applied when the client was created,
-            // are still maintained
-            [RequestHeaders.CorrelationId]: uploadCorrelationId,
+      try {
+        await apiClient.post(
+          `/persister/synchronization/jobs/${jobId}/${type as string}`,
+          {
+            [type]: batch,
           },
-        },
-      );
+          {
+            headers: {
+              // NOTE: Other headers that were applied when the client was created,
+              // are still maintained
+              [RequestHeaders.CorrelationId]: uploadCorrelationId,
+            },
+          },
+        );
+      } catch (err) {
+        cleanAxiosError(err);
+        throw err;
+      }
+
       const duration = Date.now() - startTime;
       if (duration >= 10_000) {
         logger.info(
@@ -590,3 +598,13 @@ export async function abortSynchronization({
 
   return abortedJob;
 }
+
+function cleanAxiosError(err: AxiosError) {
+  if (err.config?.headers?.Authorization) {
+    delete err.config.headers.Authorization;
+  }
+
+  if (err.config?.data) {
+    delete err.config.data;
+  }
+}