feat: add tracing to collection API

.syncpackrc

@@ -21,14 +21,21 @@
         "!dev"
       ],
       "pinVersion": "3.529.1",
-      "label": "AWS SDK Dependencies should all have the same version"
+      "label": "AWS SDK dependencies should all have the same version"
     },
     {
       "dependencies": [
-        "@sentry/**"
+        "@sentry/node"
+      ],
+      "pinVersion": "8.37.1",
+      "label": "Sentry node dependencies should all have the same version"
+    },
+    {
+      "dependencies": [
+        "@sentry/serverless"
       ],
       "pinVersion": "7.112.2",
-      "label": "AWS SDK Dependencies should all have the same version"
+      "label": "Sentry serverless (Lambda) dependencies should all have the same version"
     },
     {
       "dependencies": [

README.md

@@ -214,9 +214,61 @@ pnpm update
 pnpm build
 ```
 
-#### aws-sdk versions
+## Package Synchronization
 
-⚠ Keep aws-sdk versions in sync, because AWS sometimes introduces incompatibilities without notice.
+We use [Syncpack](https://jamiemason.github.io/syncpack/guide/getting-started/) to keep package versions consistent across servers, lambdas, and shared packages. Outside of the consistent functional expecations of using the same package version in all places, it's important to keep some package groups in sync to mitigate cross-package bugs, e.g AWS and Prisma packages.
 
-- When adding a new aws-sdk, pin it to the version used throughout the monorepo.
-- When upgrading aws-sdk, upgrade it consistently throughout the monorepo.
+The syncpack config can be found in the `./syncpackrc` file.
+
+There are two command line operations associated with Syncpack:
+
+1. `pnpm list-mismatches` will tell you if any packages are out of sync/in violation of the rules set in `.syncpackrc`. This should only happen if your current branch is changing packages/package versions. Our CI will error if the rules in `.syncpackrc` are in violation.
+
+2. `pnpm fix-mismatches` will automatically fix `package.json` files that are in violation of the rules set in `.syncpackrc` by changing package versions. This is a quick and easy way to perform an upgrade, but as with any operation that can change things at scale, be sure you check the result is what you expect.
+
+## Tracing (Servers Only - WIP)
+
+We leverage [Pocket's tracing package](https://www.npmjs.com/package/@pocket-tools/tracing) to perform traces in our `server` applications:
+
+- Collection API
+
+  - [GCP Traces](<https://console.cloud.google.com/traces/list?project=moz-fx-pocket-prod-61fb&pageState=(%22traceIntervalPicker%22:(%22groupValue%22:%22P7D%22,%22customValue%22:null),%22traceFilter%22:(%22chips%22:%22%255B%257B_22k_22_3A_22service.name_22_2C_22t_22_3A10_2C_22v_22_3A_22_5C_22collection-api_5C_22_22_2C_22s_22_3Atrue_2C_22i_22_3A_22service.name_22%257D%255D%22))>)
+  - [GCP Logs](https://cloudlogging.app.goo.gl/3Ft9tbRDo3cHfC9K7)
+  - [Unleash feature flag](https://featureflags.getpocket.dev/projects/default/features/perm.content.tracing.collections) (Dev)
+  - [Unleash feature flag](https://featureflags.readitlater.com/projects/default/features/perm.content.tracing.collections) (Prod)
+
+- Curated Corpus API (coming soon)
+- Prospect API (coming soon)
+
+Tracing is performed using Open Telemetry NPM packages that send trace data to a standalone collector ECS service in AWS, which in turn exports trace data to GCP. The Pocket tracing package also implements an
+[Open Telemetry package](https://www.npmjs.com/package/@opentelemetry/auto-instrumentations-node) that hooks into the Winston logger (which we implement via the Pocket `ts-logger` package) to auto-forward log data to GCP.
+
+### Enable/Disable Tracing in Prod & Dev
+
+Tracing can be enabled and disabled using an Unleash feature flag, which exists per service implementing tracing.
+
+To enable and configure the feature flag:
+
+1. Visit the feature flag URL for the environment in question (linked above)
+2. Toggle the `default` environment to "On" in the left hand "Enabled in environments" box
+3. Expand the `default` environment in the main, right-hand panel
+4. Click the ✎ pencil icon to edit the "Gradual rollout" strategy
+5. Move the "Rolleout" slider to 100%
+6. Click the "Variants" tab and adjust the "Payload" number to the sample rate you'd like for your traces
+   - In production, this should usually be 1% (0.01) to begin with, and can be increased slowly if needed
+7. Click "Save strategy"
+8. After some requests have been made to the service, go look at traces in GCP (using links above)
+
+To disable tracing on a service, simply toggle the `default` environment to "Off".
+
+### Local Tracing
+
+Local tracing is enabled by default and sends trace data to a Grafana Docker image. To view traces locally:
+
+1. Make sure the local service you want to trace has activity, e.g. by running a query in the Apollo Server Playground
+2. Navigate to the Grafana docker image endpoint at `http://localhost:3000/explore`
+3. Click "Explore" in the left hand menu
+4. In the dropdown at the top left of the middle pane, select "Tempo"
+5. In the main panel, select the "Service Graph" for "Query type"
+6. Click the service you want to view traces for and select "View traces"
+7. Trace away!

docker-compose.yml

@@ -52,3 +52,30 @@ services:
       retries: 30
     ports:
       - '9090:9090'
+
+  otlpcollector:
+    image: grafana/otel-lgtm
+    ports:
+      # there are lots more ports available, but these are the only one's
+      # currently being used
+      - 4317:4317 # OTLP gRPC receiver
+      - 4318:4318 # OTLP http receiver
+      - 3000:3000 # Grafana
+
+
+  # Uncomment this if you want to run the collector locally and send to GCP instead of the one used for local development
+  # Make sure to have a service json in GOOGLE_APPLICATION_CREDENTIALS_JSON in the .env file
+  # otlpcollector:
+  #   build:
+  #     context: ./servers/otel-collector
+  #     dockerfile: Dockerfile
+  #   env_file:
+  #     - .env
+  #   environment:
+  #     - DEPLOYMENT_ENVIRONMENT_NAME=local
+  #   ports:
+  #     # there are lots more ports available, but these are the only one's
+  #     # currently being used
+  #     - 4317:4317 # OTLP gRPC receiver
+  #     - 4318:4318 # OTLP http receiver
+  #     - 3000:3000 # Grafana

infrastructure/collection-api/src/config/index.ts

@@ -14,10 +14,6 @@ const rds = {
   minCapacity: isDev ? 1 : 4,
   maxCapacity: isDev ? 1 : 128,
 };
-const githubConnectionArn = isDev
-  ? 'arn:aws:codestar-connections:us-east-1:410318598490:connection/7426c139-1aa0-49e2-aabc-5aef11092032'
-  : 'arn:aws:codestar-connections:us-east-1:996905175585:connection/5fa5aa2b-a2d2-43e3-ab5a-72ececfc1870';
-const branch = isDev ? 'dev' : 'main';
 const eventBusName = `PocketEventBridge-${environment}-Shared-Event-Bus`;
 
 export const config = {
@@ -30,17 +26,17 @@ export const config = {
   domain,
   graphqlVariant,
   rds,
-  codePipeline: {
-    githubConnectionArn,
-    repository: 'pocket/collection-api',
-    branch,
-  },
   tags: {
     service: name,
     environment,
     app_code: 'pocket',
     component_code: `pocket-${name.toLowerCase()}`,
     env_code: isDev ? 'dev' : 'prod',
   },
+  tracing: {
+    url: isDev
+      ? 'https://otel-collector.getpocket.dev:443'
+      : 'https://otel-collector.readitlater.com:443',
+  },
   eventBusName,
 };

infrastructure/collection-api/src/main.ts

@@ -247,6 +247,15 @@ class CollectionAPI extends TerraformStack {
               name: 'EVENT_BUS_NAME',
               value: config.eventBusName,
             },
+            {
+              name: 'OTLP_COLLECTOR_URL',
+              value: config.tracing.url,
+            },
+            {
+              name: 'LOG_LEVEL',
+              // do not log http, graphql, or debug events
+              value: 'info',
+            },
           ],
           logGroup: this.createCustomLogGroup('app'),
           logMultilinePattern: '^\\S.+',
@@ -259,6 +268,14 @@ class CollectionAPI extends TerraformStack {
               name: 'DATABASE_URL',
               valueFrom: `${rds.secretARN}:database_url::`,
             },
+            {
+              name: 'UNLEASH_ENDPOINT',
+              valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}/UNLEASH_ENDPOINT`,
+            },
+            {
+              name: 'UNLEASH_KEY',
+              valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:${config.name}/${config.environment}/UNLEASH_KEY`,
+            },
           ],
         },
       ],
@@ -302,6 +319,8 @@ class CollectionAPI extends TerraformStack {
             resources: [
               `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/${config.name}/${config.environment}`,
               `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/${config.name}/${config.environment}/*`,
+              `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}/*`,
+              `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}`,
             ],
             effect: 'Allow',
           },
@@ -326,11 +345,6 @@ class CollectionAPI extends TerraformStack {
               'logs:CreateLogStream',
               'logs:DescribeLogStreams',
               'logs:DescribeLogGroups',
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
             ],
             resources: ['*'],
             effect: 'Allow',

packages/content-common/package.json

@@ -24,7 +24,7 @@
     "@snowplow/node-tracker": "3.5.0",
     "got": "11.8.6",
     "tslib": "2.7.0",
-    "@sentry/node": "7.112.2"
+    "@sentry/node": "8.37.1"
   },
   "devDependencies": {
     "@types/jest": "29.5.12",

packages/prospectapi-common/package.json

@@ -26,7 +26,7 @@
     "@aws-sdk/lib-dynamodb": "3.529.1",
     "@aws-sdk/util-dynamodb": "3.529.1",
     "content-common": "workspace:*",
-    "@sentry/node": "7.112.2",
+    "@sentry/node": "8.37.1",
     "cross-fetch": "3.1.5",
     "fetch-retry": "5.0.6",
     "graphql-tag": "2.12.6",