Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: split large incremental snapshots #1307

Closed
wants to merge 8 commits into from
Closed

feat: split large incremental snapshots #1307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
04e8ff3
feat: split large incremental snapshots
pauldambra Jul 16, 2024
19b75f9
account for timestamp
pauldambra Jul 16, 2024
35b1d85
after testing locally with every incremental being processed
pauldambra Jul 20, 2024
dc488af
order
pauldambra Jul 20, 2024
3f41b34
refactor
pauldambra Jul 20, 2024
54878a2
one fewer estimate
pauldambra Jul 20, 2024
eab4488
we already have a max message size, just use that
pauldambra Jul 20, 2024
ef97495
Merge branch 'main' into feat/pd/split-cinremental
pauldambra Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions functional_tests/mock-server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,11 @@ const handleRequest = (group: string) => (req: RestRequest, res: ResponseComposi
} else if (gzipCompressed) {
const data = new Uint8Array(req._body)
const decoded = strFromU8(decompressSync(data))
console.log(decoded)
body = JSON.parse(decoded)
} else {
body = JSON.parse(decodeURIComponent(body.split('=')[1]))
}
} catch (e) {
} catch {
return res(ctx.status(500))
}
}
Expand Down
290 changes: 213 additions & 77 deletions src/__tests__/extensions/replay/sessionrecording-utils.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import {
CONSOLE_LOG_PLUGIN_NAME,
ensureMaxMessageSize,
estimateSize,
FULL_SNAPSHOT_EVENT_TYPE,
PLUGIN_EVENT_TYPE,
replacementImageURI,
splitBuffer,
truncateLargeConsoleLogs,
CONSOLE_LOG_PLUGIN_NAME,
PLUGIN_EVENT_TYPE,
Expand All @@ -11,7 +16,13 @@ import {
circularReferenceReplacer,
} from '../../../extensions/replay/sessionrecording-utils'
import { largeString, threeMBAudioURI, threeMBImageURI } from '../test_data/sessionrecording-utils-test-data'
import { eventWithTime } from '@rrweb/types'
import { eventWithTime, incrementalSnapshotEvent, IncrementalSource } from '@rrweb/types'
import { serializedNodeWithId } from 'rrweb-snapshot'
import { SnapshotBuffer } from '../../../extensions/replay/sessionrecording'

const ONE_MEGABYTE = 1024 * 1024
const SEVEN_MEGABYTES = ONE_MEGABYTE * 7 * 0.9 // ~7mb (with some wiggle room)
const ONE_MEGABYTE_OF_DATA = 'a'.repeat(1024 * 1024)

describe(`SessionRecording utility functions`, () => {
describe(`filterLargeDataURLs`, () => {
Expand Down Expand Up @@ -245,93 +256,218 @@ describe(`SessionRecording utility functions`, () => {
})

describe('splitBuffer', () => {
it('should return the same buffer if size is less than SEVEN_MEGABYTES', () => {
const buffer = {
size: 5 * 1024 * 1024,
data: new Array(100).fill(0),
sessionId: 'session1',
windowId: 'window1',
}

const result = splitBuffer(buffer)
expect(result).toEqual([buffer])
})
describe('when many items in the buffer', () => {
it('should return the same buffer if size is less than the limit', () => {
const theData = new Array(100).fill(0)
const buffer = {
size: estimateSize(theData),
data: theData,
sessionId: 'session1',
windowId: 'window1',
}

it('should split the buffer into two halves if size is greater than or equal to SEVEN_MEGABYTES', () => {
const data = new Array(100).fill(0)
const expectedSize = estimateSize(new Array(50).fill(0))
const buffer = {
size: estimateSize(data),
data: data,
sessionId: 'session1',
windowId: 'window1',
}

// size limit just below the size of the buffer
const result = splitBuffer(buffer, 200)

expect(result).toHaveLength(2)
expect(result[0].data).toEqual(buffer.data.slice(0, 50))
expect(result[0].size).toEqual(expectedSize)
expect(result[1].data).toEqual(buffer.data.slice(50))
expect(result[1].size).toEqual(expectedSize)
})
const result = splitBuffer(buffer, estimateSize(theData) + 1)
expect(result).toEqual([buffer])
})

it('should split the buffer into two halves if size is greater than or equal to SEVEN_MEGABYTES', () => {
const data = new Array(100).fill(0)
const expectedSize = estimateSize(new Array(50).fill(0))
const buffer = {
size: estimateSize(data),
data: data,
sessionId: 'session1',
windowId: 'window1',
}

it('should recursively split the buffer until each part is smaller than SEVEN_MEGABYTES', () => {
const largeDataArray = new Array(100).fill('a'.repeat(1024 * 1024))
const largeDataSize = estimateSize(largeDataArray) // >100mb
const buffer = {
size: largeDataSize,
data: largeDataArray,
sessionId: 'session1',
windowId: 'window1',
}

const result = splitBuffer(buffer)

expect(result.length).toBe(20)
let partTotal = 0
let sentArray: any[] = []
result.forEach((part) => {
expect(part.size).toBeLessThan(SEVEN_MEGABYTES)
sentArray = sentArray.concat(part.data)
partTotal += part.size
// size limit just below the size of the buffer
const result = splitBuffer(buffer, 200)

expect(result).toHaveLength(2)
expect(result[0].data).toEqual(buffer.data.slice(0, 50))
expect(result[0].size).toEqual(expectedSize)
expect(result[1].data).toEqual(buffer.data.slice(50))
expect(result[1].size).toEqual(expectedSize)
})

// it's a bit bigger because we have extra square brackets and commas when stringified
expect(partTotal).toBeGreaterThan(largeDataSize)
// but not much bigger!
expect(partTotal).toBeLessThan(largeDataSize * 1.001)
// we sent the same data overall
expect(JSON.stringify(sentArray)).toEqual(JSON.stringify(largeDataArray))
})
it('should recursively split the buffer until each part is smaller than SEVEN_MEGABYTES', () => {
const largeDataArray = new Array(100).fill('a'.repeat(1024 * 1024))
const largeDataSize = estimateSize(largeDataArray) // >100mb
const buffer = {
size: largeDataSize,
data: largeDataArray,
sessionId: 'session1',
windowId: 'window1',
}

const result = splitBuffer(buffer, SEVEN_MEGABYTES)

expect(result.length).toBe(15)
let partTotal = 0
let sentArray: any[] = []
result.forEach((part) => {
expect(part.size).toBeLessThan(SEVEN_MEGABYTES * 1.2)
sentArray = sentArray.concat(part.data)
partTotal += part.size
})

// it's a bit bigger because we have extra square brackets and commas when stringified
expect(partTotal).toBeGreaterThan(largeDataSize)
// but not much bigger!
expect(partTotal).toBeLessThan(largeDataSize * 1.001)
// we sent the same data overall
expect(JSON.stringify(sentArray)).toEqual(JSON.stringify(largeDataArray))
})

it('should handle buffer with size exactly SEVEN_MEGABYTES', () => {
const buffer = {
size: SEVEN_MEGABYTES,
data: new Array(100).fill(0),
sessionId: 'session1',
windowId: 'window1',
}

const result = splitBuffer(buffer, 101)

expect(result).toHaveLength(2)
expect(result[0].data).toEqual(buffer.data.slice(0, 50))
expect(result[1].data).toEqual(buffer.data.slice(50))
})

it('should handle buffer with size exactly SEVEN_MEGABYTES', () => {
const buffer = {
size: SEVEN_MEGABYTES,
data: new Array(100).fill(0),
sessionId: 'session1',
windowId: 'window1',
}
it('should not split buffer if it has only one element', () => {
const buffer: SnapshotBuffer = {
size: estimateSize([0]),
data: [0 as unknown as eventWithTime],
sessionId: 'session1',
windowId: 'window1',
}

const result = splitBuffer(buffer)
const result = splitBuffer(buffer, estimateSize([0]) - 1)

expect(result).toHaveLength(2)
expect(result[0].data).toEqual(buffer.data.slice(0, 50))
expect(result[1].data).toEqual(buffer.data.slice(50))
expect(result).toEqual([buffer])
})
})

it('should not split buffer if it has only one element', () => {
const buffer = {
size: 10 * 1024 * 1024,
data: [0],
sessionId: 'session1',
windowId: 'window1',
}
describe('when one item in the buffer', () => {
it('should ignore full snapshots (for now)', () => {
const buffer: SnapshotBuffer = {
size: 14,
data: [{ type: '2' } as unknown as eventWithTime],
sessionId: 'session1',
windowId: 'window1',
}

const result = splitBuffer(buffer, 12)
expect(result).toEqual([buffer])
})

const result = splitBuffer(buffer)
it('should split incremental adds', () => {
const incrementalSnapshot: eventWithTime = {
type: 3,
timestamp: 12345,
data: {
source: IncrementalSource.Mutation,
adds: [
{
parentId: 1,
nextId: null,
node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId,
},
{
parentId: 2,
nextId: null,
node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId,
},
],
texts: [],
attributes: [],
// removes are processed first by the replayer, so we need to be sure we're emitting them first
removes: [{ parentId: 1, id: 2 }],
},
}
const expectedSize = estimateSize([incrementalSnapshot])
const buffer = {
size: expectedSize,
data: [incrementalSnapshot],
sessionId: 'session1',
windowId: 'window1',
}

expect(result).toEqual([buffer])
const result = splitBuffer(buffer, ONE_MEGABYTE * 0.9)
expect(result).toHaveLength(3)
const expectedSplitRemoves = [
{
timestamp: 12343,
type: 3,
data: {
// removes are processed first by the replayer, so we need to be sure we're emitting them first
removes: [{ parentId: 1, id: 2 }],
adds: [],
texts: [],
attributes: [],
source: 0,
},
} as incrementalSnapshotEvent,
]
expect(result[0]).toEqual({
...buffer,
size: estimateSize(expectedSplitRemoves),
data: expectedSplitRemoves,
})
const expectedSplitAddsOne = [
{
timestamp: 12344,
type: 3,
data: {
source: 0,
texts: [],
attributes: [],
removes: [],
adds: [
{
parentId: 1,
nextId: null,
node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId,
},
],
},
},
]
expect(result[1]).toEqual(
// the two adds each only fit one at a time, so they are split in order
// TODO if we sort these by timestamp at playback what's going to happen...
// we need to maintain the original order
{
...buffer,
size: estimateSize(expectedSplitAddsOne),
data: expectedSplitAddsOne,
}
)
const expectedSplitAddsTwo = [
{
timestamp: 12345,
type: 3,
data: {
source: 0,
texts: [],
attributes: [],
removes: [],
adds: [
{
parentId: 2,
nextId: null,
node: ONE_MEGABYTE_OF_DATA as unknown as serializedNodeWithId,
},
],
},
},
]
expect(result[2]).toEqual({
...buffer,
size: estimateSize(expectedSplitAddsTwo),
data: expectedSplitAddsTwo,
})
})
})
})

Expand Down
Loading
Loading