Skip to content

Commit

Permalink
don't always compress
Browse files Browse the repository at this point in the history
  • Loading branch information
@pauldambra
pauldambra committed Sep 30, 2024
1 parent 0013e47 commit c4b3000
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 85 deletions.
132 changes: 48 additions & 84 deletions src/__tests__/extensions/replay/sessionrecording.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,9 @@ const createIncrementalMouseEvent = () => {
})
}

const createIncrementalMutationEvent = () => {
const createIncrementalMutationEvent = (mutations?: { texts: any[] }) => {
const mutationData = {
texts: [],
texts: mutations?.texts || [],
attributes: [],
removes: [],
adds: [],
Expand All @@ -121,15 +121,15 @@ const createIncrementalMutationEvent = () => {
})
}

const createIncrementalStyleSheetEvent = () => {
const createIncrementalStyleSheetEvent = (mutations?: { adds: any[] }) => {
return createIncrementalSnapshot({
data: {
// doesn't need to be a valid style sheet event
source: 8,
id: 1,
styleId: 1,
removes: [],
adds: [],
adds: mutations.adds || [],
replace: 'something',
replaceSync: 'something',
},
Expand Down Expand Up @@ -476,9 +476,9 @@ describe('SessionRecording', () => {
const fullSnapshot = createFullSnapshot()
_emit(fullSnapshot)
expect(sessionRecording['buffer']).toEqual({
data: [{ ...fullSnapshot, cv: '2024-10', data: expect.any(String) }],
data: [fullSnapshot],
sessionId: sessionId,
size: 121,
size: 20,
windowId: 'windowId',
})
})
Expand Down Expand Up @@ -1493,22 +1493,9 @@ describe('SessionRecording', () => {

// the second snapshot remains buffered in memory
expect(sessionRecording['buffer']).toEqual({
data: [
firstSnapshotEvent,
{
...secondSnapshot,
cv: '2024-10',
data: {
...secondSnapshot.data,
adds: expect.any(String),
texts: expect.any(String),
attributes: expect.any(String),
removes: expect.any(String),
},
},
],
data: [firstSnapshotEvent, secondSnapshot],
sessionId: firstSessionId,
size: 549,
size: 186,
windowId: expect.any(String),
})

Expand All @@ -1530,22 +1517,9 @@ describe('SessionRecording', () => {
expect(posthog.capture).toHaveBeenCalledWith(
'$snapshot',
{
$snapshot_data: [
firstSnapshotEvent,
{
...secondSnapshot,
cv: '2024-10',
data: {
...secondSnapshot.data,
adds: expect.any(String),
texts: expect.any(String),
attributes: expect.any(String),
removes: expect.any(String),
},
},
],
$snapshot_data: [firstSnapshotEvent, secondSnapshot],
$session_id: firstSessionId,
$snapshot_bytes: 549,
$snapshot_bytes: 186,
$window_id: expect.any(String),
},
{
Expand Down Expand Up @@ -1605,22 +1579,9 @@ describe('SessionRecording', () => {

// the second snapshot remains buffered in memory
expect(sessionRecording['buffer']).toEqual({
data: [
firstSnapshotEvent,
{
...secondSnapshot,
cv: '2024-10',
data: {
...secondSnapshot.data,
adds: expect.any(String),
texts: expect.any(String),
attributes: expect.any(String),
removes: expect.any(String),
},
},
],
data: [firstSnapshotEvent, secondSnapshot],
sessionId: firstSessionId,
size: 549,
size: 186,
windowId: expect.any(String),
})

Expand All @@ -1646,22 +1607,9 @@ describe('SessionRecording', () => {
expect(posthog.capture).toHaveBeenCalledWith(
'$snapshot',
{
$snapshot_data: [
firstSnapshotEvent,
{
...secondSnapshot,
cv: '2024-10',
data: {
...secondSnapshot.data,
adds: expect.any(String),
texts: expect.any(String),
attributes: expect.any(String),
removes: expect.any(String),
},
},
],
$snapshot_data: [firstSnapshotEvent, secondSnapshot],
$session_id: firstSessionId,
$snapshot_bytes: 549,
$snapshot_bytes: 186,
$window_id: expect.any(String),
},
{
Expand All @@ -1685,22 +1633,9 @@ describe('SessionRecording', () => {
expect(posthog.capture).toHaveBeenCalledWith(
'$snapshot',
{
$snapshot_data: [
firstSnapshotEvent,
{
...secondSnapshot,
cv: '2024-10',
data: {
...secondSnapshot.data,
adds: expect.any(String),
texts: expect.any(String),
attributes: expect.any(String),
removes: expect.any(String),
},
},
],
$snapshot_data: [firstSnapshotEvent, secondSnapshot],
$session_id: firstSessionId,
$snapshot_bytes: 549,
$snapshot_bytes: 186,
$window_id: expect.any(String),
},
{
Expand Down Expand Up @@ -2039,7 +1974,13 @@ describe('SessionRecording', () => {
})

it('compresses full snapshot data', () => {
_emit(createFullSnapshot())
_emit(
createFullSnapshot({
data: {
content: Array(30).fill(uuidv7()).join(''),
},
})
)
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
Expand All @@ -2060,8 +2001,29 @@ describe('SessionRecording', () => {
)
})

it('does not compress small full snapshot data', () => {
_emit(createFullSnapshot({ data: { content: 'small' } }))
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
'$snapshot',
{
$snapshot_data: [
{
data: { content: 'small' },
type: 2,
},
],
$session_id: sessionId,
$snapshot_bytes: expect.any(Number),
$window_id: 'windowId',
},
captureOptions
)
})

it('compresses incremental snapshot mutation data', () => {
_emit(createIncrementalMutationEvent())
_emit(createIncrementalMutationEvent({ texts: [Array(30).fill(uuidv7()).join('')] }))
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
Expand Down Expand Up @@ -2090,7 +2052,7 @@ describe('SessionRecording', () => {
})

it('compresses incremental snapshot style data', () => {
_emit(createIncrementalStyleSheetEvent())
_emit(createIncrementalStyleSheetEvent({ adds: [Array(30).fill(uuidv7()).join('')] }))
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
Expand Down Expand Up @@ -2119,6 +2081,8 @@ describe('SessionRecording', () => {
)
})

it('does not compress small incremental snapshot data', () => {})

it('does not compress incremental snapshot non full data', () => {
const mouseEvent = createIncrementalMouseEvent()
_emit(mouseEvent)
Expand Down
10 changes: 9 additions & 1 deletion src/extensions/replay/sessionrecording.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ const BASE_ENDPOINT = '/s/'
const FIVE_MINUTES = 1000 * 60 * 5
const TWO_SECONDS = 2000
export const RECORDING_IDLE_THRESHOLD_MS = FIVE_MINUTES
export const RECORDING_MAX_EVENT_SIZE = 1024 * 1024 * 0.9 // ~1mb (with some wiggle room)
const ONE_KB = 1024
const PARTIAL_COMPRESSION_THRESHOLD = ONE_KB
export const RECORDING_MAX_EVENT_SIZE = ONE_KB * ONE_KB * 0.9 // ~1mb (with some wiggle room)
export const RECORDING_BUFFER_TIMEOUT = 2000 // 2 seconds
export const SESSION_RECORDING_BATCH_KEY = 'recordings'

Expand Down Expand Up @@ -144,6 +146,11 @@ function gzipToString(data: unknown): string {
// but we want to be able to inspect metadata during ingestion, and don't want to compress the entire event
// so we have a custom packer that only compresses part of some events
function compressEvent(event: eventWithTime, ph: PostHog): eventWithTime | compressedEventWithTime {
const originalSize = estimateSize(event)
if (originalSize < PARTIAL_COMPRESSION_THRESHOLD) {
return event
}

try {
if (event.type === EventType.FullSnapshot) {
return {
Expand Down Expand Up @@ -940,6 +947,7 @@ export class SessionRecording {
const eventToSend =
this.instance.config.session_recording.compress_events ?? true ? compressEvent(event, this.instance) : event
const size = estimateSize(eventToSend)

const properties = {
$snapshot_bytes: size,
$snapshot_data: eventToSend,
Expand Down

0 comments on commit c4b3000

Please sign in to comment.