chore: break up large rrweb_events payloads (#338)

* chore: break up large rrweb_events payloads

* chore: limit invidiual rrweb event size to 10 mb

packages/client-core/src/index.ts

@@ -19,8 +19,8 @@ export * from './EvaluationTypes';
 export * from './Hashing';
 export * from './InitializeResponse';
 export * from './Log';
-export * from './NetworkCore';
 export * from './NetworkConfig';
+export * from './NetworkCore';
 export * from './OverrideAdapter';
 export * from './ParamStoreTypes';
 export * from './SafeJs';
@@ -33,10 +33,10 @@ export * from './StatsigDataAdapter';
 export * from './StatsigEvent';
 export * from './StatsigMetadata';
 export * from './StatsigOptionsCommon';
+export * from './StatsigPlugin';
 export * from './StatsigTypeFactories';
 export * from './StatsigTypes';
 export * from './StatsigUser';
-export * from './StatsigPlugin';
 export * from './StorageProvider';
 export * from './TypedJsonParse';
 export * from './TypingUtils';

packages/combo/webpack[js-client+session-replay].config.js

@@ -5,7 +5,7 @@ const BUNDLE_FILE_NAME = 'js-client+session-replay';
 
 module.exports = createStatsigWebpackBundle({
   bundleFile: BUNDLE_FILE_NAME,
-  maxByteSize: 113_000,
+  maxByteSize: 115_000,
   dependencies: [
     '@statsig/client-core',
     '@statsig/js-client',

packages/session-replay/src/SessionReplay.ts

@@ -1,7 +1,9 @@
 import {
   ErrorBoundary,
+  Log,
   PrecomputedEvaluationsInterface,
   SDK_VERSION,
+  StatsigEvent,
   StatsigMetadataProvider,
   StatsigPlugin,
   Visibility,
@@ -10,6 +12,7 @@ import {
   _isServerEnv,
   _isUnloading,
   _subscribeToVisiblityChanged,
+  getUUID,
 } from '@statsig/client-core';
 
 import {
@@ -18,8 +21,11 @@ import {
   ReplaySessionData,
   SessionReplayClient,
 } from './SessionReplayClient';
+import { _fastApproxSizeOf } from './SizeOf';
 
-const MAX_REPLAY_PAYLOAD_BYTES = 2048;
+const REPLAY_ENQUEUE_TRIGGER_BYTES = 1024 * 10; // 10 KB
+const REPLAY_SLICE_BYTES = 1024 * 1024; // 1 MB
+const MAX_INDIVIDUAL_EVENT_BYTES = 1024 * 1024 * 10; // 10 MB
 
 type SessionReplayOptions = {
   rrwebConfig?: RRWebConfig;
@@ -28,6 +34,21 @@ type SessionReplayOptions = {
 
 type EndReason = 'is_leaving_page' | 'session_expired';
 
+type RRWebPayload = {
+  session_start_ts: string;
+  session_end_ts: string;
+  clicks_captured_cumulative: string;
+  rrweb_events: string;
+  rrweb_payload_size: string;
+  session_replay_sdk_version: string;
+  sliced_id?: string;
+  slice_index?: string;
+  slice_count?: string;
+  slice_byte_size?: string;
+  is_leaving_page?: string;
+  session_expired?: string;
+};
+
 export class StatsigSessionReplayPlugin
   implements StatsigPlugin<PrecomputedEvaluationsInterface>
 {
@@ -117,15 +138,37 @@ export class SessionReplay {
 
   private _onRecordingEvent(event: ReplayEvent, data: ReplaySessionData) {
     this._sessionData = data;
+
+    const eventApproxSize = _fastApproxSizeOf(
+      event,
+      MAX_INDIVIDUAL_EVENT_BYTES,
+    );
+
+    if (eventApproxSize > MAX_INDIVIDUAL_EVENT_BYTES) {
+      Log.warn(
+        `SessionReplay event is too large (~${eventApproxSize} bytes) and will not be logged`,
+        event,
+      );
+      return;
+    }
+
+    const approxArraySizeBefore = _fastApproxSizeOf(
+      this._events,
+      REPLAY_ENQUEUE_TRIGGER_BYTES,
+    );
     this._events.push(event);
 
-    const payload = JSON.stringify(this._events);
-    if (payload.length > MAX_REPLAY_PAYLOAD_BYTES) {
-      if (_isCurrentlyVisible()) {
-        this._bumpSessionIdleTimerAndLogRecording();
-      } else {
-        this._logRecording();
-      }
+    if (
+      approxArraySizeBefore + eventApproxSize <
+      REPLAY_ENQUEUE_TRIGGER_BYTES
+    ) {
+      return;
+    }
+
+    if (_isCurrentlyVisible()) {
+      this._bumpSessionIdleTimerAndLogRecording();
+    } else {
+      this._logRecording();
     }
   }
 
@@ -181,24 +224,36 @@ export class SessionReplay {
     }
 
     const payload = JSON.stringify(this._events);
-    const event = {
-      eventName: 'statsig::session_recording',
-      value: sessionID,
-      metadata: {
-        session_start_ts: String(data.startTime),
-        session_end_ts: String(data.endTime),
-        clicks_captured_cumulative: String(data.clickCount),
-        rrweb_events: payload,
-        rrweb_payload_size: String(payload.length),
-        session_replay_sdk_version: SDK_VERSION,
-      } as Record<string, string>,
-    };
-
-    if (endReason) {
-      event.metadata[endReason] = 'true';
-    }
+    const parts = _slicePayload(payload);
+
+    const slicedID = parts.length > 1 ? getUUID() : null;
+
+    for (let i = 0; i < parts.length; i++) {
+      const slice = parts[i];
+      const event = _makeLoggableRrwebEvent(slice, payload, sessionID, data);
+
+      if (slicedID != null) {
+        _appendSlicedMetadata(
+          event.metadata,
+          slicedID,
+          i,
+          parts.length,
+          slice.length,
+        );
+      }
+
+      if (endReason) {
+        event.metadata[endReason] = 'true';
+      }
+
+      this._client.logEvent(event);
 
-    this._client.logEvent(event);
+      if (slicedID != null) {
+        this._client.flush().catch((e) => {
+          Log.error(e);
+        });
+      }
+    }
 
     this._events = [];
   }
@@ -212,3 +267,50 @@ export class SessionReplay {
     return this._client.getContext().session.data.sessionID;
   }
 }
+
+function _slicePayload(payload: string): string[] {
+  const parts = [];
+
+  for (let i = 0; i < payload.length; i += REPLAY_SLICE_BYTES) {
+    parts.push(payload.slice(i, i + REPLAY_SLICE_BYTES));
+  }
+
+  return parts;
+}
+
+function _makeLoggableRrwebEvent(
+  slice: string,
+  payload: string,
+  sessionID: string,
+  data: ReplaySessionData,
+): StatsigEvent & { metadata: RRWebPayload } {
+  const metadata: RRWebPayload = {
+    session_start_ts: String(data.startTime),
+    session_end_ts: String(data.endTime),
+    clicks_captured_cumulative: String(data.clickCount),
+
+    rrweb_events: slice,
+    rrweb_payload_size: String(payload.length),
+
+    session_replay_sdk_version: SDK_VERSION,
+  };
+
+  return {
+    eventName: 'statsig::session_recording',
+    value: sessionID,
+    metadata,
+  };
+}
+
+function _appendSlicedMetadata(
+  metadata: RRWebPayload,
+  slicedID: string,
+  sliceIndex: number,
+  sliceCount: number,
+  sliceByteSize: number,
+) {
+  metadata.sliced_id = slicedID;
+  metadata.slice_index = String(sliceIndex);
+  metadata.slice_count = String(sliceCount);
+  metadata.slice_byte_size = String(sliceByteSize);
+}

packages/session-replay/src/SizeOf.ts

@@ -0,0 +1,28 @@
+const CURLY_AND_SQUARE_BRACKET_SIZE = 2; // [] for array, {} for object
+const APPROX_ADDITIONAL_SIZE = 1; // additional size for comma and stuff
+
+export const _fastApproxSizeOf = (
+  obj: Record<string, unknown> | Array<unknown>,
+  max: number,
+): number => {
+  let size = 0;
+  const keys = Object.keys(obj);
+  for (let i = 0; i < keys.length; i++) {
+    const key = keys[i];
+    const value = (obj as any)[key];
+    size += key.length;
+
+    if (typeof value === 'object' && value !== null) {
+      size += _fastApproxSizeOf(value, max) + CURLY_AND_SQUARE_BRACKET_SIZE;
+    } else {
+      size += String(value).length + APPROX_ADDITIONAL_SIZE;
+    }
+
+    if (size >= max) {
+      // exit early if we've exceeded the max
+      return size;
+    }
+  }
+
+  return size;
+};

packages/session-replay/src/__tests__/SessionReplaySlicing.test.ts

@@ -0,0 +1,119 @@
+import { MockRemoteServerEvalClient, anyUUID } from 'statsig-test-helpers';
+
+import { PrecomputedEvaluationsInterface } from '@statsig/client-core';
+
+import { SessionReplay } from '../SessionReplay';
+import {
+  ReplayEvent,
+  ReplaySessionData,
+  SessionReplayClient,
+} from '../SessionReplayClient';
+
+const DUMMY_DATA = 'a'.repeat(512 * 1024);
+
+const SMALL_EVENT = {
+  type: 0,
+  data: DUMMY_DATA, // 512 KB
+  timestamp: 0,
+};
+
+const LARGE_EVENT = {
+  type: 0,
+  data: DUMMY_DATA + DUMMY_DATA + DUMMY_DATA, // 1.5 MB
+  timestamp: 0,
+};
+
+describe('Session Replay Force', () => {
+  let client: jest.MockedObject<PrecomputedEvaluationsInterface>;
+  let sessionReplay: SessionReplay;
+  let replayer: SessionReplayClient;
+  let emitter: (latest: ReplayEvent, data: ReplaySessionData) => void;
+
+  async function emitEvent(event: ReplayEvent) {
+    emitter(event, {
+      startTime: 0,
+      endTime: 0,
+      clickCount: 0,
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 1)); // allow promises to resolve
+  }
+
+  beforeAll(() => {
+    client = MockRemoteServerEvalClient.create();
+    client.flush.mockResolvedValue();
+
+    const ctx = {
+      errorBoundary: { wrap: jest.fn() },
+      values: { session_recording_rate: 1, can_record_session: true },
+      session: { data: { sessionID: 'my-session-id' } },
+    } as any;
+
+    client.getContext.mockReturnValue(ctx);
+
+    sessionReplay = new SessionReplay(client);
+    replayer = (sessionReplay as any)._replayer;
+
+    const original = replayer.record.bind(replayer);
+    replayer.record = (cb: any, config: any, stopCallback: any) => {
+      emitter = cb;
+      return original(cb, config, stopCallback);
+    };
+
+    replayer.stop();
+    sessionReplay.forceStartRecording();
+  });
+
+  beforeEach(() => {
+    client.logEvent.mockClear();
+  });
+
+  it('logs a single event when below max payload size', async () => {
+    await emitEvent(SMALL_EVENT);
+
+    expect(client.logEvent).toHaveBeenCalledTimes(1);
+    expect(client.flush).not.toHaveBeenCalled();
+  });
+
+  it('logs sliced events when above max payload size', async () => {
+    await emitEvent(LARGE_EVENT);
+
+    expect(client.logEvent).toHaveBeenCalledTimes(2);
+    expect(client.flush).toHaveBeenCalledTimes(2);
+  });
+
+  describe('Sliced events payload', () => {
+    beforeEach(async () => {
+      await emitEvent(LARGE_EVENT);
+    });
+
+    it('logs the first slice with the correct metadata', () => {
+      const metadata = (client.logEvent.mock.calls[0][0] as any).metadata;
+
+      expect(metadata).toMatchObject({
+        sliced_id: anyUUID(),
+        slice_index: '0',
+        slice_count: '2',
+        slice_byte_size: '1048576',
+      });
+    });
+
+    it('logs the second slice with the correct metadata', () => {
+      const metadata = (client.logEvent.mock.calls[1][0] as any).metadata;
+
+      expect(metadata).toMatchObject({
+        sliced_id: anyUUID(),
+        slice_index: '1',
+        slice_count: '2',
+        slice_byte_size: '524324',
+      });
+    });
+
+    it('logs the same slice id across the two slices', () => {
+      const metadata1 = (client.logEvent.mock.calls[0][0] as any).metadata;
+      const metadata2 = (client.logEvent.mock.calls[1][0] as any).metadata;
+
+      expect(metadata1.sliced_id).toEqual(metadata2.sliced_id);
+    });
+  });
+});