instrument

config.yaml

@@ -0,0 +1,34 @@
+---
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+exporters:
+  debug:
+    verbosity: detailed
+  otlp:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+
+processors:
+  batch:
+
+service:
+  extensions: [zpages]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, otlp]
+  telemetry:
+    metrics:
+      address: 0.0.0.0:8888
+    logs:
+      level: info
+
+extensions:
+  zpages:
+    endpoint: :55679

docker-compose.yaml

@@ -0,0 +1,17 @@
+version: "2.1"
+services:
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    ports:
+      - 16686:16686
+  collector:
+    image: otel/opentelemetry-collector-contrib:0.88.0
+    command: [ "--config=/etc/otel/config.yaml" ]
+    ports:
+      - 8006:8006
+      - 8888:8888
+      - 55679:55679
+      - 4317:4317
+      - 4318:4318
+    volumes:
+      - ./config.yaml:/etc/otel/config.yaml

package.json

@@ -37,9 +37,9 @@
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-jest": "^27.2.3",
     "husky": "^4.2.5",
-    "jest": "^29.6.1",
+    "jest": "^29.7.0",
     "jest-extended": "^4.0.0",
-    "lerna": "^7.1.4",
+    "lerna": "^7.4.2",
     "lint-staged": "^10.2.6",
     "prettier": "^3.0.0",
     "ts-jest": "^29.1.1",

packages/integration-sdk-cli/package.json

@@ -27,13 +27,19 @@
     "@jupiterone/data-model": "^0.54.0",
     "@jupiterone/integration-sdk-core": "^11.3.1",
     "@jupiterone/integration-sdk-runtime": "^11.3.1",
+    "@opentelemetry/api": "^1.7.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.45.1",
+    "@opentelemetry/instrumentation": "^0.45.1",
+    "@opentelemetry/resources": "^1.18.1",
+    "@opentelemetry/sdk-trace-node": "^1.18.1",
+    "@opentelemetry/semantic-conventions": "^1.18.1",
     "chalk": "^4",
     "commander": "^9.4.0",
     "fs-extra": "^10.1.0",
     "globby": "^11.0.0",
     "inquirer-checkbox-plus-prompt": "^1.4.2",
     "js-yaml": "^4.1.0",
-    "json-diff": "^0.5.4",
+    "json-diff": "^1.0.6",
     "lodash": "^4.17.19",
     "markdown-table": "^2.0.0",
     "neo4j-driver": "^4.3.3",
@@ -44,6 +50,7 @@
   },
   "devDependencies": {
     "@jupiterone/integration-sdk-private-test-utils": "^11.3.1",
+    "@opentelemetry/auto-instrumentations-node": "^0.40.1",
     "@pollyjs/adapter-node-http": "^6.0.5",
     "@pollyjs/core": "^6.0.5",
     "@pollyjs/persister-fs": "^6.0.5",

packages/integration-sdk-cli/src/commands/collect.ts

@@ -17,6 +17,15 @@ import {
   configureRuntimeFilesystem,
 } from './options';
 
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import {
+  BatchSpanProcessor,
+  NodeTracerProvider,
+} from '@opentelemetry/sdk-trace-node';
+import { Resource } from '@opentelemetry/resources';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
+import { registerInstrumentations } from '@opentelemetry/instrumentation';
+
 // coercion function to collect multiple values for a flag
 const collector = (value: string, arr: string[]) => {
   arr.push(...value.split(','));
@@ -84,6 +93,31 @@ export function collect() {
 
       const enableSchemaValidation = !options.disableSchemaValidation;
 
+      const resource = Resource.default().merge(
+        new Resource({
+          [SemanticResourceAttributes.SERVICE_NAME]: 'integration-sdk',
+          [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0',
+        }),
+      );
+      const tracerProvider = new NodeTracerProvider({
+        resource,
+      });
+      const exporter = new OTLPTraceExporter({
+        url:
+          'http://' +
+          (process.env.OTEL_COLLECTOR_URL ?? 'localhost:4318') +
+          '/v1/traces',
+      });
+      const processor = new BatchSpanProcessor(exporter);
+      tracerProvider.addSpanProcessor(processor);
+      tracerProvider.register();
+      registerInstrumentations({
+        instrumentations: [
+          getNoodeAutoInstrumentations();
+        ]
+      })
+
+      const tracer = tracerProvider.getTracer('tracer');
       const results = await executeIntegrationLocally(
         config,
         {
@@ -95,9 +129,17 @@ export function collect() {
           enableSchemaValidation,
           graphObjectStore,
           pretty: !options.noPretty,
+          tracer,
         },
       );
 
+      await tracerProvider.forceFlush();
+      await tracerProvider.shutdown();
+      await processor.forceFlush();
+      await exporter.forceFlush();
+
+      await exporter.shutdown();
+
       log.displayExecutionResults(results);
     });
 }

packages/integration-sdk-cli/src/commands/run.ts

@@ -28,6 +28,13 @@ import {
   validateApiClientOptions,
   validateSyncOptions,
 } from './options';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import {
+  BatchSpanProcessor,
+  NodeTracerProvider,
+} from '@opentelemetry/sdk-trace-node';
+import { Resource } from '@opentelemetry/resources';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
 
 const DEFAULT_UPLOAD_CONCURRENCY = 5;
 
@@ -92,6 +99,27 @@ export function run(): Command {
 
       try {
         const enableSchemaValidation = !options.disableSchemaValidation;
+        const resource = Resource.default().merge(
+          new Resource({
+            [SemanticResourceAttributes.SERVICE_NAME]: 'integration-sdk-run',
+            [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0',
+          }),
+        );
+        const tracerProvider = new NodeTracerProvider({
+          resource,
+        });
+        const exporter = new OTLPTraceExporter({
+          url:
+            'http://' +
+            (process.env.OTEL_COLLECTOR_URL ?? 'localhost:4318') +
+            '/v1/traces',
+        });
+        const processor = new BatchSpanProcessor(exporter);
+        tracerProvider.addSpanProcessor(processor);
+        tracerProvider.register();
+
+        const tracer = tracerProvider.getTracer('tracer');
+
         const executionResults = await executeIntegrationInstance(
           logger,
           createIntegrationInstanceForLocalExecution(invocationConfig),
@@ -102,6 +130,7 @@ export function run(): Command {
             },
           },
           {
+            tracer,
             enableSchemaValidation,
             graphObjectStore,
             createStepGraphObjectDataUploader(stepId) {
@@ -134,6 +163,12 @@ export function run(): Command {
             'Synchronization finalization result.',
           );
         }
+        await tracerProvider.forceFlush();
+        await tracerProvider.shutdown();
+        await processor.forceFlush();
+        await exporter.forceFlush();
+
+        await exporter.shutdown();
       } catch (err) {
         await eventPublishingQueue.onIdle();
         if (!logger.isHandledError(err)) {

packages/integration-sdk-core/src/types/context.ts

@@ -51,12 +51,14 @@ export type IntegrationInstanceExecutionContext<
 export type IntegrationExecutionContext<
   TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig,
   TExecutionConfig extends
-    IntegrationExecutionConfig = IntegrationExecutionConfig,
+  IntegrationExecutionConfig = IntegrationExecutionConfig,
 > = IntegrationLoadExecutionConfigContext<TConfig> & {
   executionConfig: TExecutionConfig;
 };
 
 export type StepExecutionContext = ExecutionContext & {
+  tracer: <T>(name: string, fn: () => Promise<T>) => Promise<T>;
+
   jobState: JobState;
 };
 
@@ -69,6 +71,6 @@ export type StepExecutionContext = ExecutionContext & {
 export interface IntegrationStepExecutionContext<
   TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig,
   TExecutionConfig extends
-    IntegrationExecutionConfig = IntegrationExecutionConfig,
+  IntegrationExecutionConfig = IntegrationExecutionConfig,
 > extends IntegrationExecutionContext<TConfig, TExecutionConfig>,
-    StepExecutionContext {}
+  StepExecutionContext { }

packages/integration-sdk-runtime/package.json

@@ -41,6 +41,8 @@
   },
   "devDependencies": {
     "@jupiterone/integration-sdk-private-test-utils": "^11.3.1",
+    "@opentelemetry/api": "^1.7.0",
+    "@opentelemetry/sdk-trace-base": "^1.18.1",
     "get-port": "^5.1.1",
     "memfs": "^3.2.0",
     "ts-node": "^9.1.0",

packages/integration-sdk-runtime/src/api/index.ts

@@ -8,6 +8,7 @@ import {
   IntegrationApiKeyRequiredError,
 } from './error';
 import { gzipData } from '../synchronization/util';
+import path from 'path';
 
 export type ApiClient = Alpha;
 
@@ -67,10 +68,37 @@ export function createApiClient({
     // AlphaInterceptors _must_ return the config or a Promise for the config.
     client.interceptors.request.use(compressRequest);
   }
+  const x = {
+    async post(url: string, data: Record<string, any>, config: AlphaOptions) {
+      const loc = path.join(apiBaseUrl, url);
+      const response = await fetch(loc, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify(data),
+      });
+
+      if (!response.ok) {
+        const data = await response.text();
+        console.log({ data });
+        throw new Error('bad response');
+      }
+      return response.json();
+    },
+    async get(url: string) {
+      const loc = path.join(apiBaseUrl, url);
+      const response = await fetch(loc);
+      if (!response.ok) {
+        throw new Error('bad response');
+      }
+      return await response.json();
+    },
+  };
+  return x as ApiClient;
+
   return client;
 }
 
-export const compressRequest: AlphaInterceptor = async function (
+export const compressRequest: AlphaInterceptor = async function(
   config: AlphaOptions,
 ) {
   if (

packages/integration-sdk-runtime/src/execution/dependencyGraph.ts

@@ -33,6 +33,7 @@ import {
   DuplicateEntityReport,
   DuplicateKeyTracker,
 } from './duplicateKeyTracker';
+import { withTracing } from './tracer';
 
 /**
  * This function accepts a list of steps and constructs a dependency graph
@@ -561,6 +562,8 @@ function buildStepContext<
       stepId,
     }),
     jobState,
+    // @ts-ignore
+    tracer: withTracing,
   };
 
   return stepExecutionContext as TStepExecutionContext;