feat: capture network payloads (internal alpha)

cypress/e2e/session-recording.cy.js

@@ -43,8 +43,7 @@ describe('Session recording', () => {
                     cy.phCaptures({ full: true }).then((captures) => {
                         // should be a pageview and a $snapshot
                         expect(captures.map((c) => c.event)).to.deep.equal(['$pageview', '$snapshot'])
-                        // the amount of captured data should be deterministic
-                        // but of course that would be too easy
+
                         expect(captures[1]['properties']['$snapshot_data']).to.have.length.above(33).and.below(38)
                         // a meta and then a full snapshot
                         expect(captures[1]['properties']['$snapshot_data'][0].type).to.equal(4) // meta

src/__tests__/extensions/replay/config.test.ts

@@ -0,0 +1,80 @@
+import { defaultConfig } from '../../../posthog-core'
+import { buildNetworkRequestOptions } from '../../../extensions/replay/config'
+
+describe('config', () => {
+    describe('network request options', () => {
+        describe('maskRequestFn', () => {
+            it('can enable header recording remotely', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), { recordHeaders: true })
+                expect(networkOptions.recordHeaders).toBe(true)
+                expect(networkOptions.recordBody).toBe(undefined)
+            })
+
+            it('can enable body recording remotely', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), { recordBody: true })
+                expect(networkOptions.recordHeaders).toBe(undefined)
+                expect(networkOptions.recordBody).toBe(true)
+            })
+
+            it('client can force disable recording', () => {
+                const posthogConfig = defaultConfig()
+                posthogConfig.session_recording.recordHeaders = false
+                posthogConfig.session_recording.recordBody = false
+                const networkOptions = buildNetworkRequestOptions(posthogConfig, {
+                    recordHeaders: true,
+                    recordBody: true,
+                })
+                expect(networkOptions.recordHeaders).toBe(false)
+                expect(networkOptions.recordBody).toBe(false)
+            })
+
+            it('should remove the Authorization header from requests even if no other config is set', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), {})
+                const cleaned = networkOptions.maskRequestFn!({
+                    url: 'something',
+                    requestHeaders: {
+                        Authorization: 'Bearer 123',
+                        'content-type': 'application/json',
+                    },
+                })
+                expect(cleaned?.requestHeaders).toEqual({
+                    'content-type': 'application/json',
+                })
+            })
+
+            it('should cope with no headers when even if no other config is set', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), {})
+                const cleaned = networkOptions.maskRequestFn!({
+                    url: 'something',
+                    requestHeaders: undefined,
+                })
+                expect(cleaned?.requestHeaders).toBeUndefined()
+            })
+
+            it('should remove the Authorization header from requests even when a mask request fn is set', () => {
+                const posthogConfig = defaultConfig()
+                posthogConfig.session_recording.maskNetworkRequestFn = (data) => {
+                    return {
+                        ...data,
+                        requestHeaders: {
+                            ...(data.requestHeaders ? data.requestHeaders : {}),
+                            'content-type': 'edited',
+                        },
+                    }
+                }
+                const networkOptions = buildNetworkRequestOptions(posthogConfig, {})
+
+                const cleaned = networkOptions.maskRequestFn!({
+                    url: 'something',
+                    requestHeaders: {
+                        Authorization: 'Bearer 123',
+                        'content-type': 'application/json',
+                    },
+                })
+                expect(cleaned?.requestHeaders).toEqual({
+                    'content-type': 'edited',
+                })
+            })
+        })
+    })
+})

src/extensions/replay/config.ts

@@ -0,0 +1,71 @@
+import { NetworkRecordOptions, NetworkRequest, PostHogConfig } from '../../types'
+import { _isFunction } from '../../utils/type-utils'
+
+export const defaultNetworkOptions: NetworkRecordOptions = {
+    initiatorTypes: [
+        'audio',
+        'beacon',
+        'body',
+        'css',
+        'early-hint',
+        'embed',
+        'fetch',
+        'frame',
+        'iframe',
+        'icon',
+        'image',
+        'img',
+        'input',
+        'link',
+        'navigation',
+        'object',
+        'ping',
+        'script',
+        'track',
+        'video',
+        'xmlhttprequest',
+    ],
+    maskRequestFn: (data: NetworkRequest) => data,
+    recordHeaders: false,
+    recordBody: false,
+    recordInitialRequests: false,
+}
+
+const removeAuthorizationHeader = (data: NetworkRequest): NetworkRequest => {
+    delete data.requestHeaders?.['Authorization']
+    return data
+}
+
+/**
+ *  whether a maskRequestFn is provided or not,
+ *  we ensure that we remove the Authorization header from requests
+ *  we _never_ want to record that header by accident
+ *  if someone complains then we'll add an opt-in to let them override it
+ */
+export const buildNetworkRequestOptions = (
+    instanceConfig: PostHogConfig,
+    remoteNetworkOptions: Pick<NetworkRecordOptions, 'recordHeaders' | 'recordBody'>
+): NetworkRecordOptions => {
+    const config = instanceConfig.session_recording as NetworkRecordOptions
+    // client can always disable despite remote options
+    const canRecordHeaders = config.recordHeaders === false ? false : remoteNetworkOptions.recordHeaders
+    const canRecordBody = config.recordBody === false ? false : remoteNetworkOptions.recordBody
+
+    config.maskRequestFn = _isFunction(instanceConfig.session_recording.maskNetworkRequestFn)
+        ? (data) => {
+              const cleanedRequest = removeAuthorizationHeader(data)
+              return instanceConfig.session_recording.maskNetworkRequestFn?.(cleanedRequest) ?? undefined
+          }
+        : undefined
+
+    if (!config.maskRequestFn) {
+        config.maskRequestFn = removeAuthorizationHeader
+    }
+
+    return {
+        ...defaultNetworkOptions,
+        ...config,
+        recordHeaders: canRecordHeaders,
+        recordBody: canRecordBody,
+    }
+}

src/extensions/replay/sessionrecording.ts

@@ -14,13 +14,15 @@ import {
     truncateLargeConsoleLogs,
 } from './sessionrecording-utils'
 import { PostHog } from '../../posthog-core'
-import { DecideResponse, NetworkRequest, Properties } from '../../types'
+import { DecideResponse, NetworkRecordOptions, NetworkRequest, Properties } from '../../types'
 import { EventType, type eventWithTime, type listenerHandler } from '@rrweb/types'
 import Config from '../../config'
 import { _timestamp, loadScript } from '../../utils'
 
-import { _isBoolean, _isNull, _isNumber, _isObject, _isString, _isUndefined } from '../../utils/type-utils'
+import { _isBoolean, _isFunction, _isNull, _isNumber, _isObject, _isString, _isUndefined } from '../../utils/type-utils'
 import { logger } from '../../utils/logger'
+import { window } from '../../utils/globals'
+import { buildNetworkRequestOptions } from './config'
 
 const BASE_ENDPOINT = '/s/'
 
@@ -90,6 +92,7 @@ export class SessionRecording {
     private receivedDecide: boolean
     private rrwebRecord: rrwebRecord | undefined
     private isIdle = false
+    private _networkPayloadCapture: Pick<NetworkRecordOptions, 'recordHeaders' | 'recordBody'> | undefined = undefined
 
     private _linkedFlagSeen: boolean = false
     private _lastActivityTimestamp: number = Date.now()
@@ -252,6 +255,8 @@ export class SessionRecording {
             })
         }
 
+        this._networkPayloadCapture = response.sessionRecording?.networkPayloadCapture
+
         const receivedSampleRate = response.sessionRecording?.sampleRate
         this._sampleRate =
             _isUndefined(receivedSampleRate) || _isNull(receivedSampleRate) ? null : parseFloat(receivedSampleRate)
@@ -462,14 +467,26 @@ export class SessionRecording {
                 },
             })
 
+        const plugins = []
+
+        if ((window as any).rrwebConsoleRecord && this.isConsoleLogCaptureEnabled) {
+            plugins.push((window as any).rrwebConsoleRecord.getRecordConsolePlugin())
+        }
+        if (this._networkPayloadCapture) {
+            if (_isFunction((window as any).getRecordNetworkPlugin)) {
+                plugins.push(
+                    (window as any).getRecordNetworkPlugin(
+                        buildNetworkRequestOptions(this.instance.config, this._networkPayloadCapture)
+                    )
+                )
+            }
+        }
+
         this.stopRrweb = this.rrwebRecord({
             emit: (event) => {
                 this.onRRwebEmit(event)
             },
-            plugins:
-                (window as any).rrwebConsoleRecord && this.isConsoleLogCaptureEnabled
-                    ? [(window as any).rrwebConsoleRecord.getRecordConsolePlugin()]
-                    : [],
+            plugins,
             ...sessionRecordingOptions,
         })
 
@@ -550,6 +567,8 @@ export class SessionRecording {
                 url,
             }
 
+            // TODO we should deprecate this and use the same function for this masking and the rrweb/network plugin
+            // TODO or deprecate this and provide a new clearer name so this would be `maskURLPerformanceFn` or similar
             networkRequest = userSessionRecordingOptions.maskNetworkRequestFn(networkRequest)
 
             return networkRequest?.url