feat: Add replay capture vector config+docker image and deployment (#…

…24264)

.github/workflows/replay-capture.yml

@@ -0,0 +1,35 @@
+name: Vector Replay Capture Tests
+
+on:
+    workflow_dispatch:
+    pull_request:
+        paths:
+            - vector/**
+            - .github/workflows/replay-capture.yml
+
+    workflow_call:
+
+jobs:
+    vector-test:
+        name: Vector test
+        runs-on: ubuntu-20.04
+        env:
+            QUOTA_LIMITED_TEAMS_PATH: vector/replay-capture/tests/quota_limited_teams.csv
+            OVERFLOW_SESSIONS_PATH: vector/replay-capture/tests/overflow_sessions.csv
+            KAFKA_BOOSTRAP_SERVERS: dummy:9092
+            KAFKA_EVENTS_TOPIC: session_recording_snapshot_item_events
+            KAFKA_OVERFLOW_TOPIC: session_recording_snapshot_item_overflow
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Install Vector
+              run: |
+                  wget https://github.com/vectordotdev/vector/releases/download/v0.40.0/vector-0.40.0-x86_64-unknown-linux-gnu.tar.gz
+                  tar xzvf vector-0.40.0-x86_64-unknown-linux-gnu.tar.gz ./vector-x86_64-unknown-linux-gnu/bin/vector
+                  sudo mv ./vector-x86_64-unknown-linux-gnu/bin/vector /usr/bin/vector
+
+            - name: Run vector tests
+              run: |
+                  yq -i e 'explode(.)' vector/replay-capture/vector.yaml
+                  vector test vector/replay-capture/*.yaml

.github/workflows/vector-docker-build-deploy.yml

@@ -0,0 +1,105 @@
+name: Build and deploy replay capture container images
+
+on:
+    workflow_dispatch:
+    push:
+        paths:
+            - 'vector/**'
+            - '.github/workflows/vector-docker-build-deploy.yml'
+        branches:
+            - 'master'
+
+jobs:
+    build:
+        name: Build and publish container image
+        runs-on: depot-ubuntu-22.04-4
+        permissions:
+            id-token: write # allow issuing OIDC tokens for this workflow run
+            contents: read # allow reading the repo contents
+            packages: write # allow push to ghcr.io
+
+        defaults:
+            run:
+                working-directory: vector/
+
+        steps:
+            - name: Check Out Repo
+              # Checkout project code
+              # Use sparse checkout to only select files in vector directory
+              # Turning off cone mode ensures that files in the project root are not included during checkout
+              uses: actions/checkout@v4
+              with:
+                  sparse-checkout: 'vector/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Set up Depot CLI
+              uses: depot/setup-action@v1
+
+            - name: Login to DockerHub
+              uses: docker/login-action@v2
+              with:
+                  username: posthog
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+            - name: Login to ghcr.io
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+                  logout: false
+
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+
+            - name: Docker meta
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                  images: ghcr.io/posthog/posthog/replay-capture
+                  tags: |
+                      type=ref,event=pr
+                      type=ref,event=branch
+                      type=semver,pattern={{version}}
+                      type=semver,pattern={{major}}.{{minor}}
+                      type=sha
+
+            - name: Set up Docker Buildx
+              id: buildx
+              uses: docker/setup-buildx-action@v2
+
+            - name: Build and push image
+              id: docker_build
+              uses: depot/build-push-action@v1
+              with:
+                  context: ./vector/replay-capture/
+                  file: ./vector/replay-capture/Dockerfile
+                  push: true
+                  tags: ${{ steps.meta.outputs.tags }}
+                  labels: ${{ steps.meta.outputs.labels }}
+                  platforms: linux/arm64
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
+                  build-args: BIN=${{ matrix.image }}
+
+            - name: Container image digest
+              run: echo ${{ steps.docker_build.outputs.digest }}
+
+            - name: Trigger replay capture deployment
+              uses: peter-evans/repository-dispatch@v3
+              with:
+                  token: ${{ steps.deployer.outputs.token }}
+                  repository: PostHog/charts
+                  event-type: commit_state_update
+                  client-payload: |
+                      {
+                        "values": {
+                          "image": {
+                            "sha": "${{ steps.docker_build.outputs.digest }}"
+                          }
+                        },
+                        "release": "replay-capture-vector",
+                        "commit": ${{ toJson(github.event.head_commit) }},
+                        "repository": ${{ toJson(github.repository) }},
+                        "labels": []
+                      }

docker-compose.base.yml

@@ -105,14 +105,34 @@ services:
         restart: on-failure
 
     capture:
-        image: ghcr.io/posthog/capture:main
+        image: ghcr.io/posthog/posthog/capture:master
         restart: on-failure
         environment:
             ADDRESS: '0.0.0.0:3000'
             KAFKA_TOPIC: 'events_plugin_ingestion'
             KAFKA_HOSTS: 'kafka:9092'
             REDIS_URL: 'redis://redis:6379/'
 
+    replay-capture:
+        image: ghcr.io/posthog/posthog/replay-capture:master
+        build:
+            context: vector/replay-capture
+        restart: on-failure
+        entrypoint: ['sh', '-c']
+        command:
+            - |
+                set -x
+                # seed empty required data files
+                mkdir -p /etc/vector/data
+                echo "token" > /etc/vector/data/quota_limited_teams.csv
+                echo "session_id" > /etc/vector/data/overflow_sessions.csv
+                exec vector -v --watch-config
+        environment:
+            KAFKA_EVENTS_TOPIC: session_recording_snapshot_item_events
+            KAFKA_OVERFLOW_TOPIC: session_recording_snapshot_item_overflow
+            KAFKA_BOOSTRAP_SERVERS: 'kafka:9092'
+            REDIS_URL: 'redis://redis:6379/'
+
     plugins:
         command: ./bin/plugin-server --no-restart-loop
         restart: on-failure

docker-compose.dev.yml

@@ -117,6 +117,17 @@ services:
             - redis
             - kafka
 
+    # Optional capture
+    replay-capture:
+        extends:
+            file: docker-compose.base.yml
+            service: replay-capture
+        ports:
+            - 3001:8000
+        depends_on:
+            - redis
+            - kafka
+
     livestream:
         extends:
             file: docker-compose.base.yml

vector/replay-capture/Dockerfile

@@ -0,0 +1,12 @@
+FROM alpine as config-builder
+
+RUN apk add -U yq
+
+WORKDIR /config
+COPY vector.yaml .
+# evaluate with yq, basically to expand anchors (which vector doesn't support)
+RUN yq -i e 'explode(.)' vector.yaml
+
+FROM timberio/vector:0.40.X-alpine
+
+COPY --from=config-builder /config/vector.yaml /etc/vector/vector.yaml

vector/replay-capture/tests.yaml

@@ -0,0 +1,76 @@
+tests:
+    - name: Basic Test
+      inputs:
+          - insert_at: quota_check
+            type: vrl
+            source: |
+                .message = [{}]
+                .message[0].properties = {}
+                .message[0].properties."$$session_id" = "123"
+                .message[0].properties."$$window_id" = "123"
+                .message[0].properties."token" = "123"
+                .message[0].properties."distinct_id" = "123"
+                .message[0].properties."$$snapshot_data" = [{"offset": 123}]
+
+                .ip = "0.0.0.0"
+                .timestamp = now()
+                ."_" = "123456789"
+                %token = "123"
+
+      outputs:
+          - conditions:
+                - source: |
+                      assert!(is_string(.uuid))
+                      assert!(is_string(%headers.token))
+                      assert!(is_string(parse_json!(.data).uuid))
+                      assert!(parse_json!(.data).properties."$$snapshot_items"[0].offset == 123)
+                  type: vrl
+            extract_from: overflow_check._unmatched
+    - name: Quota limited
+      inputs:
+          - insert_at: quota_check
+            type: vrl
+            source: |
+                .message = [{}]
+                .message[0].properties = {}
+                .message[0].properties."$$session_id" = "123"
+                .message[0].properties."$$window_id" = "123"
+                .message[0].properties."token" = "limited_token"
+                .message[0].properties."distinct_id" = "123"
+                .message[0].properties."$$snapshot_data" = [{"offset": 123}]
+
+                .ip = "0.0.0.0"
+                .timestamp = now()
+                ."_" = "123456789"
+                %token = "limited_token"
+
+      outputs:
+          - conditions:
+                - source: |
+                      true
+                  type: vrl
+            extract_from: metric_quota_dropped
+    - name: Overflow
+      inputs:
+          - insert_at: quota_check
+            type: vrl
+            source: |
+                .message = [{}]
+                .message[0].properties = {}
+                .message[0].properties."$$session_id" = "overflow_session"
+                .message[0].properties."$$window_id" = "123"
+                .message[0].properties."token" = "123"
+                .message[0].properties."distinct_id" = "123"
+                .message[0].properties."$$snapshot_data" = [{"offset": 123}]
+
+                .ip = "0.0.0.0"
+                .timestamp = now()
+                ."_" = "123456789"
+                %token = "123"
+
+      outputs:
+          - conditions:
+                - source: |
+                      true
+                  type: vrl
+            extract_from: overflow_check.overflow

vector/replay-capture/tests/overflow_sessions.csv

@@ -0,0 +1,2 @@
+session_id
+overflow_session

vector/replay-capture/tests/quota_limited_teams.csv

@@ -0,0 +1,2 @@
+token
+limited_token