Merge pull request #951 from JupiterOne/halo-161-logging-improvements

Logging Improvement / Error Updates
JupiterOne · Sep 18, 2023 · 2da7049 · 2da7049
2 parents 472ca01 + c36beae
commit 2da7049
Show file tree

Hide file tree

Showing 23 changed files with 122 additions and 211 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,12 @@ and this project adheres to
 
 ## Unreleased
 
+## 10.5.3 - 2023-09-18
+
+- Use the same logger for all logs created via the `j1-integration run` command
+  to improve formatting consistency
+- Update internal `err` log serializer
+
 ## 10.5.2 - 2023-09-14
 
 - Improvements to integration graph generator

diff --git a/lerna.json b/lerna.json
@@ -4,5 +4,5 @@
     "packages/integration-sdk-*",
     "packages/cli"
   ],
-  "version": "10.5.2"
+  "version": "10.5.3"
 }
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/cli",
-  "version": "10.5.2",
+  "version": "10.5.3",
   "description": "The JupiterOne cli",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -24,8 +24,8 @@
     "test": "jest"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-core": "^10.5.2",
-    "@jupiterone/integration-sdk-runtime": "^10.5.2",
+    "@jupiterone/integration-sdk-core": "^10.5.3",
+    "@jupiterone/integration-sdk-runtime": "^10.5.3",
     "@lifeomic/attempt": "^3.0.3",
     "commander": "^5.0.0",
     "globby": "^11.0.1",

diff --git a/packages/integration-sdk-benchmark/package.json b/packages/integration-sdk-benchmark/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-benchmark",
-  "version": "10.5.2",
+  "version": "10.5.3",
   "private": true,
   "description": "SDK benchmarking scripts",
   "main": "./src/index.js",
@@ -15,8 +15,8 @@
     "benchmark": "for file in ./src/benchmarks/*; do yarn prebenchmark && node $file; done"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-core": "^10.5.2",
-    "@jupiterone/integration-sdk-runtime": "^10.5.2",
+    "@jupiterone/integration-sdk-core": "^10.5.3",
+    "@jupiterone/integration-sdk-runtime": "^10.5.3",
     "benchmark": "^2.1.4"
   }
 }
diff --git a/packages/integration-sdk-cli/package.json b/packages/integration-sdk-cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-cli",
-  "version": "10.5.2",
+  "version": "10.5.3",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -25,8 +25,8 @@
   },
   "dependencies": {
     "@jupiterone/data-model": "^0.54.0",
-    "@jupiterone/integration-sdk-core": "^10.5.2",
-    "@jupiterone/integration-sdk-runtime": "^10.5.2",
+    "@jupiterone/integration-sdk-core": "^10.5.3",
+    "@jupiterone/integration-sdk-runtime": "^10.5.3",
     "chalk": "^4",
     "commander": "^9.4.0",
     "fs-extra": "^10.1.0",
@@ -44,7 +44,7 @@
     "vis": "^4.21.0-EOL"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^10.5.2",
+    "@jupiterone/integration-sdk-private-test-utils": "^10.5.3",
     "@pollyjs/adapter-node-http": "^6.0.5",
     "@pollyjs/core": "^6.0.5",
     "@pollyjs/persister-fs": "^6.0.5",

diff --git a/packages/integration-sdk-cli/src/__tests__/cli-run-failure.test.ts b/packages/integration-sdk-cli/src/__tests__/cli-run-failure.test.ts
@@ -2,11 +2,9 @@ import { Polly } from '@pollyjs/core';
 import NodeHttpAdapter from '@pollyjs/adapter-node-http';
 import FSPersister from '@pollyjs/persister-fs';
 import { loadProjectStructure } from '@jupiterone/integration-sdk-private-test-utils';
-import { SynchronizationJobStatus } from '@jupiterone/integration-sdk-core';
 import { generateSynchronizationJob } from './util/synchronization';
 import { createCli } from '../index';
 import { setupSynchronizerApi } from './util/synchronization';
-import * as log from '../log';
 import { createTestPolly } from './util/recording';
 
 jest.mock('../log');
@@ -37,17 +35,20 @@ test('aborts synchronization job if an error occurs', async () => {
 
   setupSynchronizerApi({ polly, job, baseUrl: 'https://api.us.jupiterone.io' });
 
+  let calledAbort = false;
+  polly.server
+    .post(
+      `https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/abort`,
+    )
+    .intercept((req, res) => {
+      calledAbort = true;
+    });
   await createCli().parseAsync([
     'node',
     'j1-integration',
     'run',
     '--integrationInstanceId',
     'test',
   ]);
-
-  expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
-  expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
-    ...job,
-    status: SynchronizationJobStatus.ABORTED,
-  });
+  expect(calledAbort).toBe(true);
 });
diff --git a/packages/integration-sdk-cli/src/__tests__/cli-run.test.ts b/packages/integration-sdk-cli/src/__tests__/cli-run.test.ts
@@ -10,12 +10,6 @@ import {
   setupSynchronizerApi,
 } from './util/synchronization';
 
-import {
-  StepResultStatus,
-  SynchronizationJobStatus,
-} from '@jupiterone/integration-sdk-core';
-
-import * as log from '../log';
 import { createTestPolly } from './util/recording';
 
 jest.mock('../log');
@@ -80,92 +74,19 @@ test('disables graph object schema validation', async () => {
   expect(process.env.ENABLE_GRAPH_OBJECT_SCHEMA_VALIDATION).toBeUndefined();
 });
 
-test('step should fail if enableSchemaValidation = true', async () => {
-  loadProjectStructure('instanceWithNonValidatingSteps');
-  const job = generateSynchronizationJob();
-
-  setupSynchronizerApi({
-    polly,
-    job,
-    baseUrl: 'https://api.us.jupiterone.io',
-  });
-
-  await createCli().parseAsync([
-    'node',
-    'j1-integration',
-    'run',
-    '--integrationInstanceId',
-    'test',
-  ]);
-
-  expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
-
-  expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
-  expect(log.displayExecutionResults).toHaveBeenCalledWith({
-    integrationStepResults: [
-      {
-        id: 'fetch-users',
-        name: 'Fetch Users',
-        declaredTypes: ['my_user'],
-        partialTypes: [],
-        encounteredTypes: [],
-        status: StepResultStatus.FAILURE,
-      },
-    ],
-    metadata: {
-      partialDatasets: {
-        types: ['my_user'],
-      },
-    },
-  });
-});
-
-test('step should pass if enableSchemaValidation = false', async () => {
-  loadProjectStructure('instanceWithNonValidatingSteps');
-  const job = generateSynchronizationJob();
-
-  setupSynchronizerApi({
-    polly,
-    job,
-    baseUrl: 'https://api.us.jupiterone.io',
-  });
-
-  await createCli().parseAsync([
-    'node',
-    'j1-integration',
-    'run',
-    '--integrationInstanceId',
-    'test',
-    '--disable-schema-validation',
-  ]);
-
-  expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
-
-  expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
-  expect(log.displayExecutionResults).toHaveBeenCalledWith({
-    integrationStepResults: [
-      {
-        id: 'fetch-users',
-        name: 'Fetch Users',
-        declaredTypes: ['my_user'],
-        partialTypes: [],
-        encounteredTypes: ['my_user'],
-        status: StepResultStatus.SUCCESS,
-      },
-    ],
-    metadata: {
-      partialDatasets: {
-        types: [],
-      },
-    },
-  });
-});
-
 test('executes integration and performs upload', async () => {
   const job = generateSynchronizationJob();
 
   setupSynchronizerApi({ polly, job, baseUrl: 'https://api.us.jupiterone.io' });
 
+  let calledFinalize = false;
+  polly.server
+    .post(
+      `https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/finalize`,
+    )
+    .intercept((req, res) => {
+      calledFinalize = true;
+    });
   await createCli().parseAsync([
     'node',
     'j1-integration',
@@ -174,43 +95,7 @@ test('executes integration and performs upload', async () => {
     'test',
   ]);
 
-  expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
-  expect(log.displayExecutionResults).toHaveBeenCalledWith({
-    integrationStepResults: [
-      {
-        id: 'fetch-accounts',
-        name: 'Fetch Accounts',
-        declaredTypes: ['my_account'],
-        partialTypes: [],
-        encounteredTypes: ['my_account'],
-        status: StepResultStatus.SUCCESS,
-      },
-      {
-        id: 'fetch-users',
-        name: 'Fetch Users',
-        declaredTypes: ['my_user', 'my_account_has_user'],
-        partialTypes: [],
-        encounteredTypes: ['my_user', 'my_account_has_user'],
-        status: StepResultStatus.SUCCESS,
-      },
-    ],
-    metadata: {
-      partialDatasets: {
-        types: [],
-      },
-    },
-  });
-
-  expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
-  expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
-    ...job,
-    status: SynchronizationJobStatus.FINALIZE_PENDING,
-    // These are the expected number of entities and relationships
-    // collected when executing the
-    // 'typeScriptIntegrationProject' fixture
-    numEntitiesUploaded: 2,
-    numRelationshipsUploaded: 1,
-  });
+  expect(calledFinalize).toBe(true);
 });
 
 test('executes integration and skips finalization with skip-finalize', async () => {
@@ -222,6 +107,15 @@ test('executes integration and skips finalization with skip-finalize', async ()
     baseUrl: 'https://api.us.jupiterone.io',
   });
 
+  let calledFinalize = false;
+  polly.server
+    .post(
+      `https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/finalize`,
+    )
+    .intercept((req, res) => {
+      calledFinalize = true;
+    });
+
   await createCli().parseAsync([
     'node',
     'j1-integration',
@@ -230,19 +124,7 @@ test('executes integration and skips finalization with skip-finalize', async ()
     'test',
     '--skip-finalize',
   ]);
-
-  expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
-
-  expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
-  expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
-    ...job,
-    status: SynchronizationJobStatus.AWAITING_UPLOADS,
-    // These are the expected number of entities and relationships
-    // collected when executing the
-    // 'typeScriptIntegrationProject' fixture
-    numEntitiesUploaded: 2,
-    numRelationshipsUploaded: 1,
-  });
+  expect(calledFinalize).toBe(false);
 });
 
 test('does not publish events for source "api" since there is no integrationJobId', async () => {
@@ -256,7 +138,9 @@ test('does not publish events for source "api" since there is no integrationJobI
 
   let eventsPublished = false;
   polly.server
-    .post(`https://example.com/persister/synchronization/jobs/${job.id}/events`)
+    .post(
+      `https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/events`,
+    )
     .intercept((req, res) => {
       eventsPublished = true;
     });
@@ -272,8 +156,6 @@ test('does not publish events for source "api" since there is no integrationJobI
   ]);
 
   expect(eventsPublished).toBe(false);
-  expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
-  expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
 });
 
 test('should use JUPITERONE_API_KEY value in Authorization request header', async () => {

diff --git a/packages/integration-sdk-cli/src/__tests__/util/synchronization.ts b/packages/integration-sdk-cli/src/__tests__/util/synchronization.ts
@@ -97,8 +97,14 @@ export function generateSynchronizationJob(
     id: 'test',
     source: options?.source || 'integration-managed',
     scope: options?.scope,
-    integrationJobId: options?.integrationJobId || 'test-job-id',
-    integrationInstanceId: options?.integrationInstanceId || 'test-instance-id',
+    integrationJobId:
+      options?.source === 'api'
+        ? undefined
+        : options?.integrationJobId || 'test-job-id',
+    integrationInstanceId:
+      options?.source === 'api'
+        ? undefined
+        : options?.integrationInstanceId || 'test-instance-id',
     status: SynchronizationJobStatus.AWAITING_UPLOADS,
     startTimestamp: Date.now(),
     numEntitiesUploaded: 0,