Enable batching multiple events into a single POST request over buffe…

…r size if maxPostBytes setting is followed
snowplow · Oct 17, 2024 · 7a9c12a · 7a9c12a
1 parent 0b97285
commit 7a9c12a
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 20 deletions.
diff --git a/libraries/tracker-core/src/emitter/emitter_request.ts b/libraries/tracker-core/src/emitter/emitter_request.ts
@@ -48,7 +48,6 @@ export interface EmitterRequestConfiguration {
   keepalive?: boolean;
   postPath?: string;
   useStm?: boolean;
-  bufferSize?: number;
   maxPostBytes?: number,
   credentials?: 'omit' | 'same-origin' | 'include';
 }
@@ -89,7 +88,6 @@ export function newEmitterRequest({
   keepalive = true,
   postPath = '/com.snowplowanalytics.snowplow/tp2',
   useStm = true,
-  bufferSize,
   maxPostBytes = 40000,
   credentials = 'include',
 }: EmitterRequestConfiguration): EmitterRequest {
@@ -128,9 +126,6 @@ export function newEmitterRequest({
 
   function isFull(): boolean {
     if (usePost) {
-      if (bufferSize !== undefined && countEvents() >= Math.max(1, bufferSize)) {
-        return true;
-      }
       return countBytes() >= maxPostBytes;
     } else {
       return events.length >= 1;

diff --git a/libraries/tracker-core/src/emitter/index.ts b/libraries/tracker-core/src/emitter/index.ts
@@ -47,6 +47,7 @@ export interface EmitterConfigurationBase {
   bufferSize?: number;
   /**
    * The max size a POST request can be before the tracker will force send it
+   * Also dictates the max size of a POST request before a batch of events is split into multiple requests
    * @defaultValue 40000
    */
   maxPostBytes?: number;
@@ -312,7 +313,6 @@ export function newEmitter({
       customHeaders,
       connectionTimeout,
       keepalive,
-      bufferSize,
       maxPostBytes,
       useStm,
       credentials,

diff --git a/libraries/tracker-core/test/emitter/emitter_request.test.ts b/libraries/tracker-core/test/emitter/emitter_request.test.ts
@@ -6,7 +6,7 @@ import { newEventStorePayload } from '../../src/event_store_payload';
 
 const newEmitterEventFromPayload = (payload: Record<string, unknown>) => {
   return newEmitterEvent(newEventStorePayload({ payload }));
-}
+};
 
 // MARK: - addEvent
 
@@ -89,34 +89,31 @@ test('countEvents returns the correct event count', (t) => {
 
 // MARK: - isFull
 
-test('isFull returns false when not reached buffer size', (t) => {
+test('isFull returns false when not reached max post bytes', (t) => {
   const request = newEmitterRequest({
     endpoint: 'https://example.com',
     maxPostBytes: 1000,
-    bufferSize: 2,
   });
 
   t.true(request.addEvent(newEmitterEventFromPayload({ e: 'pv', p: 'web' })));
   t.false(request.isFull());
 });
 
-test('isFull returns true when reached buffer size', (t) => {
+test('isFull returns false when reached buffer size and not max post bytes', (t) => {
   const request = newEmitterRequest({
     endpoint: 'https://example.com',
     maxPostBytes: 1000,
-    bufferSize: 2,
   });
 
   t.true(request.addEvent(newEmitterEventFromPayload({ e: 'pv', p: 'web' })));
   t.true(request.addEvent(newEmitterEventFromPayload({ e: 'pv', p: 'mob' })));
-  t.true(request.isFull());
+  t.false(request.isFull());
 });
 
 test('isFull returns true when reached max post bytes', (t) => {
   const request = newEmitterRequest({
     endpoint: 'https://example.com',
     maxPostBytes: 10,
-    bufferSize: 2,
   });
 
   t.true(request.addEvent(newEmitterEventFromPayload({ e: 'pv', p: 'web' })));

diff --git a/trackers/node-tracker/test/tracker.ts b/trackers/node-tracker/test/tracker.ts
@@ -244,11 +244,13 @@ for (const eventMethod of testMethods) {
             eventMethod,
             bufferSize: 0,
             onRequestSuccess: (batch) => {
-              const expected = batch[0]['e'] === 'tr' ? expectedTransaction : expectedItem;
+              batch.forEach((payload) => {
+                const expected = payload['e'] === 'tr' ? expectedTransaction : expectedItem;
 
-              checkPayload(batch[0], expected, t);
+                checkPayload(payload, expected, t);
 
-              requestCount--;
+                requestCount--;
+              });
               if (requestCount === 0) {
                 resolve(batch);
               }
@@ -472,11 +474,13 @@ for (const eventMethod of testMethods) {
         endpoint,
         eventMethod,
         bufferSize: 0,
-        onRequestSuccess: ([pl]) => {
-          checkPayload(pl, expected, t);
-          count--;
+        onRequestSuccess: (batch) => {
+          batch.forEach((pl) => {
+            checkPayload(pl, expected, t);
+            count--;
+          });
           if (count === 0) {
-            resolve(pl);
+            resolve(batch);
           }
         },
         onRequestFailure: reject,