diff --git a/functional_tests/mock-server.ts b/functional_tests/mock-server.ts index eea0ebbfe..a3b32c1bd 100644 --- a/functional_tests/mock-server.ts +++ b/functional_tests/mock-server.ts @@ -25,12 +25,11 @@ const handleRequest = (group: string) => (req: RestRequest, res: ResponseComposi } else if (gzipCompressed) { const data = new Uint8Array(req._body) const decoded = strFromU8(decompressSync(data)) - console.log(decoded) body = JSON.parse(decoded) } else { body = JSON.parse(decodeURIComponent(body.split('=')[1])) } - } catch (e) { + } catch { return res(ctx.status(500)) } } diff --git a/src/__tests__/extensions/replay/sessionrecording-utils.test.ts b/src/__tests__/extensions/replay/sessionrecording-utils.test.ts index d0faef5b1..43e8d9995 100644 --- a/src/__tests__/extensions/replay/sessionrecording-utils.test.ts +++ b/src/__tests__/extensions/replay/sessionrecording-utils.test.ts @@ -1,6 +1,11 @@ import { + CONSOLE_LOG_PLUGIN_NAME, ensureMaxMessageSize, + estimateSize, + FULL_SNAPSHOT_EVENT_TYPE, + PLUGIN_EVENT_TYPE, replacementImageURI, + splitBuffer, truncateLargeConsoleLogs, CONSOLE_LOG_PLUGIN_NAME, PLUGIN_EVENT_TYPE, @@ -11,7 +16,13 @@ import { circularReferenceReplacer, } from '../../../extensions/replay/sessionrecording-utils' import { largeString, threeMBAudioURI, threeMBImageURI } from '../test_data/sessionrecording-utils-test-data' -import { eventWithTime } from '@rrweb/types' +import { eventWithTime, incrementalSnapshotEvent, IncrementalSource } from '@rrweb/types' +import { serializedNodeWithId } from 'rrweb-snapshot' +import { SnapshotBuffer } from '../../../extensions/replay/sessionrecording' + +const ONE_MEGABYTE = 1024 * 1024 +const SEVEN_MEGABYTES = ONE_MEGABYTE * 7 * 0.9 // ~7mb (with some wiggle room) +const ONE_MEGABYTE_OF_DATA = 'a'.repeat(1024 * 1024) describe(`SessionRecording utility functions`, () => { describe(`filterLargeDataURLs`, () => { @@ -245,93 +256,218 @@ describe(`SessionRecording utility functions`, () => { }) describe('splitBuffer', () => { - it('should return the same buffer if size is less than SEVEN_MEGABYTES', () => { - const buffer = { - size: 5 * 1024 * 1024, - data: new Array(100).fill(0), - sessionId: 'session1', - windowId: 'window1', - } - - const result = splitBuffer(buffer) - expect(result).toEqual([buffer]) - }) + describe('when many items in the buffer', () => { + it('should return the same buffer if size is less than the limit', () => { + const theData = new Array(100).fill(0) + const buffer = { + size: estimateSize(theData), + data: theData, + sessionId: 'session1', + windowId: 'window1', + } - it('should split the buffer into two halves if size is greater than or equal to SEVEN_MEGABYTES', () => { - const data = new Array(100).fill(0) - const expectedSize = estimateSize(new Array(50).fill(0)) - const buffer = { - size: estimateSize(data), - data: data, - sessionId: 'session1', - windowId: 'window1', - } - - // size limit just below the size of the buffer - const result = splitBuffer(buffer, 200) - - expect(result).toHaveLength(2) - expect(result[0].data).toEqual(buffer.data.slice(0, 50)) - expect(result[0].size).toEqual(expectedSize) - expect(result[1].data).toEqual(buffer.data.slice(50)) - expect(result[1].size).toEqual(expectedSize) - }) + const result = splitBuffer(buffer, estimateSize(theData) + 1) + expect(result).toEqual([buffer]) + }) + + it('should split the buffer into two halves if size is greater than or equal to SEVEN_MEGABYTES', () => { + const data = new Array(100).fill(0) + const expectedSize = estimateSize(new Array(50).fill(0)) + const buffer = { + size: estimateSize(data), + data: data, + sessionId: 'session1', + windowId: 'window1', + } - it('should recursively split the buffer until each part is smaller than SEVEN_MEGABYTES', () => { - const largeDataArray = new Array(100).fill('a'.repeat(1024 * 1024)) - const largeDataSize = estimateSize(largeDataArray) // >100mb - const buffer = { - size: largeDataSize, - data: largeDataArray, - sessionId: 'session1', - windowId: 'window1', - } - - const result = splitBuffer(buffer) - - expect(result.length).toBe(20) - let partTotal = 0 - let sentArray: any[] = [] - result.forEach((part) => { - expect(part.size).toBeLessThan(SEVEN_MEGABYTES) - sentArray = sentArray.concat(part.data) - partTotal += part.size + // size limit just below the size of the buffer + const result = splitBuffer(buffer, 200) + + expect(result).toHaveLength(2) + expect(result[0].data).toEqual(buffer.data.slice(0, 50)) + expect(result[0].size).toEqual(expectedSize) + expect(result[1].data).toEqual(buffer.data.slice(50)) + expect(result[1].size).toEqual(expectedSize) }) - // it's a bit bigger because we have extra square brackets and commas when stringified - expect(partTotal).toBeGreaterThan(largeDataSize) - // but not much bigger! - expect(partTotal).toBeLessThan(largeDataSize * 1.001) - // we sent the same data overall - expect(JSON.stringify(sentArray)).toEqual(JSON.stringify(largeDataArray)) - }) + it('should recursively split the buffer until each part is smaller than SEVEN_MEGABYTES', () => { + const largeDataArray = new Array(100).fill('a'.repeat(1024 * 1024)) + const largeDataSize = estimateSize(largeDataArray) // >100mb + const buffer = { + size: largeDataSize, + data: largeDataArray, + sessionId: 'session1', + windowId: 'window1', + } + + const result = splitBuffer(buffer, SEVEN_MEGABYTES) + + expect(result.length).toBe(15) + let partTotal = 0 + let sentArray: any[] = [] + result.forEach((part) => { + expect(part.size).toBeLessThan(SEVEN_MEGABYTES * 1.2) + sentArray = sentArray.concat(part.data) + partTotal += part.size + }) + + // it's a bit bigger because we have extra square brackets and commas when stringified + expect(partTotal).toBeGreaterThan(largeDataSize) + // but not much bigger! + expect(partTotal).toBeLessThan(largeDataSize * 1.001) + // we sent the same data overall + expect(JSON.stringify(sentArray)).toEqual(JSON.stringify(largeDataArray)) + }) + + it('should handle buffer with size exactly SEVEN_MEGABYTES', () => { + const buffer = { + size: SEVEN_MEGABYTES, + data: new Array(100).fill(0), + sessionId: 'session1', + windowId: 'window1', + } + + const result = splitBuffer(buffer, 101) + + expect(result).toHaveLength(2) + expect(result[0].data).toEqual(buffer.data.slice(0, 50)) + expect(result[1].data).toEqual(buffer.data.slice(50)) + }) - it('should handle buffer with size exactly SEVEN_MEGABYTES', () => { - const buffer = { - size: SEVEN_MEGABYTES, - data: new Array(100).fill(0), - sessionId: 'session1', - windowId: 'window1', - } + it('should not split buffer if it has only one element', () => { + const buffer: SnapshotBuffer = { + size: estimateSize([0]), + data: [0 as unknown as eventWithTime], + sessionId: 'session1', + windowId: 'window1', + } - const result = splitBuffer(buffer) + const result = splitBuffer(buffer, estimateSize([0]) - 1) - expect(result).toHaveLength(2) - expect(result[0].data).toEqual(buffer.data.slice(0, 50)) - expect(result[1].data).toEqual(buffer.data.slice(50)) + expect(result).toEqual([buffer]) + }) }) - it('should not split buffer if it has only one element', () => { - const buffer = { - size: 10 * 1024 * 1024, - data: [0], - sessionId: 'session1', - windowId: 'window1', - } + describe('when one item in the buffer', () => { + it('should ignore full snapshots (for now)', () => { + const buffer: SnapshotBuffer = { + size: 14, + data: [{ type: '2' } as unknown as eventWithTime], + sessionId: 'session1', + windowId: 'window1', + } + + const result = splitBuffer(buffer, 12) + expect(result).toEqual([buffer]) + }) - const result = splitBuffer(buffer) + it('should split incremental adds', () => { + const incrementalSnapshot: eventWithTime = { + type: 3, + timestamp: 12345, + data: { + source: IncrementalSource.Mutation, + adds: [ + { + parentId: 1, + nextId: null, + node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId, + }, + { + parentId: 2, + nextId: null, + node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId, + }, + ], + texts: [], + attributes: [], + // removes are processed first by the replayer, so we need to be sure we're emitting them first + removes: [{ parentId: 1, id: 2 }], + }, + } + const expectedSize = estimateSize([incrementalSnapshot]) + const buffer = { + size: expectedSize, + data: [incrementalSnapshot], + sessionId: 'session1', + windowId: 'window1', + } - expect(result).toEqual([buffer]) + const result = splitBuffer(buffer, ONE_MEGABYTE * 0.9) + expect(result).toHaveLength(3) + const expectedSplitRemoves = [ + { + timestamp: 12343, + type: 3, + data: { + // removes are processed first by the replayer, so we need to be sure we're emitting them first + removes: [{ parentId: 1, id: 2 }], + adds: [], + texts: [], + attributes: [], + source: 0, + }, + } as incrementalSnapshotEvent, + ] + expect(result[0]).toEqual({ + ...buffer, + size: estimateSize(expectedSplitRemoves), + data: expectedSplitRemoves, + }) + const expectedSplitAddsOne = [ + { + timestamp: 12344, + type: 3, + data: { + source: 0, + texts: [], + attributes: [], + removes: [], + adds: [ + { + parentId: 1, + nextId: null, + node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId, + }, + ], + }, + }, + ] + expect(result[1]).toEqual( + // the two adds each only fit one at a time, so they are split in order + // TODO if we sort these by timestamp at playback what's going to happen... + // we need to maintain the original order + { + ...buffer, + size: estimateSize(expectedSplitAddsOne), + data: expectedSplitAddsOne, + } + ) + const expectedSplitAddsTwo = [ + { + timestamp: 12345, + type: 3, + data: { + source: 0, + texts: [], + attributes: [], + removes: [], + adds: [ + { + parentId: 2, + nextId: null, + node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId, + }, + ], + }, + }, + ] + expect(result[2]).toEqual({ + ...buffer, + size: estimateSize(expectedSplitAddsTwo), + data: expectedSplitAddsTwo, + }) + }) }) }) diff --git a/src/extensions/replay/sessionrecording-utils.ts b/src/extensions/replay/sessionrecording-utils.ts index a08ba24ca..d00106fb1 100644 --- a/src/extensions/replay/sessionrecording-utils.ts +++ b/src/extensions/replay/sessionrecording-utils.ts @@ -1,17 +1,19 @@ -import type { +import { blockClass, eventWithTime, hooksParam, KeepIframeSrcFn, listenerHandler, maskTextClass, + mutationCallbackParam, + mutationData, pluginEvent, RecordPlugin, SamplingStrategy, } from '@rrweb/types' import type { DataURLOptions, MaskInputFn, MaskInputOptions, MaskTextFn, Mirror, SlimDOMOptions } from 'rrweb-snapshot' -import { isObject } from '../../utils/type-utils' +import { isNullish, isObject } from '../../utils/type-utils' import { SnapshotBuffer } from './sessionrecording' // taken from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#circular_references @@ -162,29 +164,122 @@ export function truncateLargeConsoleLogs(_event: eventWithTime) { return _event } -export const SEVEN_MEGABYTES = 1024 * 1024 * 7 * 0.9 // ~7mb (with some wiggle room) +function sliceList(list: any[], sizeLimit: number): any[][] { + const size = estimateSize(list) + if (size < sizeLimit) { + return [list] + } else { + const times = Math.ceil(size / sizeLimit) + const chunkLength = Math.ceil(list.length / times) + const chunks = [] + for (let i = 0; i < list.length; i += chunkLength) { + chunks.push(list.slice(i, i + chunkLength)) + } + return chunks + } +} + +function sliceBuffer(buffer: SnapshotBuffer, sizeLimit: number): SnapshotBuffer[] { + const bufferChunks = sliceList(buffer.data, sizeLimit) + return bufferChunks.map((data) => ({ + size: estimateSize(data), + data, + sessionId: buffer.sessionId, + windowId: buffer.windowId, + })) +} + +function hasIncrementalContent(e: eventWithTime): boolean { + const mutationData = e.data as mutationData + return ( + !isNullish(mutationData) && + (!!mutationData?.adds?.length || + !!mutationData?.removes?.length || + !!mutationData?.texts?.length || + !!mutationData?.attributes?.length) + ) +} + +function countChildren(xs: any[][]): number { + return xs.reduce((acc, x) => acc + x.length, 0) +} + +function incrementalSnapshotFrom(mutationData: Partial, timestamp: number) { + return { + type: INCREMENTAL_SNAPSHOT_EVENT_TYPE, + data: { + source: MUTATION_SOURCE_TYPE, + removes: [], + adds: [], + texts: [], + attributes: [], + ...mutationData, + }, + timestamp: timestamp, + } +} + +function splitIncrementalData(bufferedData: eventWithTime, sizeLimit: number): eventWithTime[] { + // NB: this isn't checking the size so will _always_ split incremental snapshots + if (bufferedData.type === INCREMENTAL_SNAPSHOT_EVENT_TYPE && bufferedData.data.source === MUTATION_SOURCE_TYPE) { + // so at this point we know that the buffer is too large, and we have a single incremental snapshot + // it may be that a single item in the buffer is too large + // or that there are many small items of one or more types that end up being too large + // rrweb processes removes, then adds, then texts, the attributes, + // so we can split them in that order + const bufferedMutations = bufferedData.data as mutationData + const removes = sliceList(bufferedMutations.removes || [], sizeLimit) + const adds = sliceList(bufferedMutations.adds || [], sizeLimit) + const texts = sliceList(bufferedMutations.texts || [], sizeLimit) + const attributes = sliceList(bufferedMutations.attributes || [], sizeLimit) + + // the incoming data has a single timestamp, so we need to adjust the timestamps of the split data, + // so we count how many children we have in total, and then we adjust the timestamp + // so that if there are 10 the first item is 9 milliseconds before the original timestamp + // and the final item has the original timestamp + const alteration = + countChildren(removes) + countChildren(adds) + countChildren(texts) + countChildren(attributes) + let timestampWiggleMarker = 1 -// recursively splits large buffers into smaller ones -// uses a pretty high size limit to avoid splitting too much -export function splitBuffer(buffer: SnapshotBuffer, sizeLimit: number = SEVEN_MEGABYTES): SnapshotBuffer[] { - if (buffer.size >= sizeLimit && buffer.data.length > 1) { - const half = Math.floor(buffer.data.length / 2) - const firstHalf = buffer.data.slice(0, half) - const secondHalf = buffer.data.slice(half) return [ - splitBuffer({ - size: estimateSize(firstHalf), - data: firstHalf, - sessionId: buffer.sessionId, - windowId: buffer.windowId, - }), - splitBuffer({ - size: estimateSize(secondHalf), - data: secondHalf, - sessionId: buffer.sessionId, - windowId: buffer.windowId, - }), - ].flatMap((x) => x) + ...removes.map((remove) => + incrementalSnapshotFrom( + { removes: remove }, + bufferedData.timestamp - alteration + timestampWiggleMarker++ + ) + ), + ...adds.map((add) => + incrementalSnapshotFrom({ adds: add }, bufferedData.timestamp - alteration + timestampWiggleMarker++) + ), + ...texts.map((text) => + incrementalSnapshotFrom({ texts: text }, bufferedData.timestamp - alteration + timestampWiggleMarker++) + ), + ...attributes.map((attribute) => + incrementalSnapshotFrom( + { attributes: attribute }, + bufferedData.timestamp - alteration + timestampWiggleMarker++ + ) + ), + ].filter(hasIncrementalContent) + } else { + return [bufferedData] + } +} + +// uses a pretty high size limit to avoid splitting too much +export function splitBuffer(buffer: SnapshotBuffer, sizeLimit: number = MAX_MESSAGE_SIZE): SnapshotBuffer[] { + if (buffer.size >= sizeLimit) { + // it may be because one or more incremental snapshots is very large + const splitData = buffer.data.map((bd) => splitIncrementalData(bd, sizeLimit)).flat() + const splitBuffer: SnapshotBuffer = { + // NB this is no longer totally accurate but will be replaced in sliceBuffer below + size: buffer.size, + data: splitData, + sessionId: buffer.sessionId, + windowId: buffer.windowId, + } + // or because the array of snapshots in the buffer is now too large + return sliceBuffer(splitBuffer, sizeLimit) } else { return [buffer] } diff --git a/src/extensions/replay/sessionrecording.ts b/src/extensions/replay/sessionrecording.ts index 14caeb2fe..2adc30b58 100644 --- a/src/extensions/replay/sessionrecording.ts +++ b/src/extensions/replay/sessionrecording.ts @@ -73,7 +73,7 @@ type SessionRecordingStatus = 'disabled' | 'sampled' | 'active' | 'buffering' export interface SnapshotBuffer { size: number - data: any[] + data: eventWithTime[] sessionId: string windowId: string }