feat(transformation): adding rudder libraries support (#1817)

* feat(transformation): adding rudder libraries support
rudderlabs · Feb 21, 2023 · 1c91d22 · 1c91d22
1 parent 5be8891
commit 1c91d22
Show file tree

Hide file tree

Showing 3 changed files with 110 additions and 7 deletions.
diff --git a/src/util/customTransforrmationsStore-v1.js b/src/util/customTransforrmationsStore-v1.js
@@ -5,11 +5,13 @@ const { responseStatusHandler } = require('./utils');
 
 const transformationCache = {};
 const libraryCache = {};
+const rudderLibraryCache = {};
 
-// const CONFIG_BACKEND_URL = "http://localhost:5000";
+// const CONFIG_BACKEND_URL = 'http://localhost:5000';
 const CONFIG_BACKEND_URL = process.env.CONFIG_BACKEND_URL || 'https://api.rudderlabs.com';
 const getTransformationURL = `${CONFIG_BACKEND_URL}/transformation/getByVersionId`;
 const getLibrariesUrl = `${CONFIG_BACKEND_URL}/transformationLibrary/getByVersionId`;
+const getRudderLibrariesUrl = `${CONFIG_BACKEND_URL}/rudderstackTransformationLibraries`;
 
 // Gets the transformation from config backend.
 // Stores the transformation object in memory with time to live after which it expires.
@@ -64,4 +66,31 @@ async function getLibraryCodeV1(versionId) {
   }
 }
 
-module.exports = { getTransformationCodeV1, getLibraryCodeV1 };
+async function getRudderLibByImportName(importName) {
+  const rudderLibrary = rudderLibraryCache[importName];
+  if (rudderLibrary) return rudderLibrary;
+  const tags = {
+    libraryVersionId: importName,
+    version: 1,
+    type: 'rudderlibrary',
+  };
+  try {
+    const [name, version] = importName.split('/').slice(-2);
+    const url = `${getRudderLibrariesUrl}/${name}?version=${version}`;
+    const startTime = new Date();
+    const response = await fetchWithProxy(url);
+
+    responseStatusHandler(response.status, 'Rudder Library', importName, url);
+    stats.increment('get_libraries_code.success', tags);
+    stats.timing('get_libraries_code', startTime, tags);
+    const myJson = await response.json();
+    rudderLibraryCache[importName] = myJson;
+    return myJson;
+  } catch (error) {
+    logger.error(error);
+    stats.increment('get_libraries_code.error', tags);
+    throw error;
+  }
+}
+
+module.exports = { getTransformationCodeV1, getLibraryCodeV1, getRudderLibByImportName };
diff --git a/src/util/ivmFactory.js b/src/util/ivmFactory.js
@@ -3,10 +3,12 @@ const fetch = require('node-fetch');
 const _ = require('lodash');
 
 const stats = require('./stats');
-const { getLibraryCodeV1 } = require('./customTransforrmationsStore-v1');
+const { getLibraryCodeV1, getRudderLibByImportName } = require('./customTransforrmationsStore-v1');
 const { parserForImport } = require('./parser');
 const logger = require('../logger');
 
+const RUDDER_LIBRARY_REGEX = /^@rs\/[A-Za-z]+\/v[0-9]{1,3}$/;
+
 const isolateVmMem = 128;
 async function evaluateModule(isolate, context, moduleCode) {
   const module = await isolate.compileModule(moduleCode);
@@ -25,18 +27,29 @@ async function createIvm(code, libraryVersionIds, versionId, testMode) {
   const createIvmStartTime = new Date();
   const logs = [];
   const libraries = await Promise.all(
-    libraryVersionIds.map(async (libraryVersionId) => getLibraryCodeV1(libraryVersionId)),
+    libraryVersionIds.map(async (libraryVersionId) => await getLibraryCodeV1(libraryVersionId)),
   );
   const librariesMap = {};
   if (code && libraries) {
-    const extractedLibraries = Object.keys(await parserForImport(code));
-    // TODO: Check if this should this be &&
+    const extractedLibImportNames = Object.keys(await parserForImport(code));
+
     libraries.forEach((library) => {
       const libHandleName = _.camelCase(library.name);
-      if (extractedLibraries.includes(libHandleName)) {
+      if (extractedLibImportNames.includes(libHandleName)) {
         librariesMap[libHandleName] = library.code;
       }
     });
+
+    // Extract ruddder libraries from import names
+    const rudderLibImportNames = extractedLibImportNames.filter((name) =>
+      RUDDER_LIBRARY_REGEX.test(name),
+    );
+    const rudderLibraries = await Promise.all(
+      rudderLibImportNames.map(async (importName) => await getRudderLibByImportName(importName)),
+    );
+    rudderLibraries.forEach((library) => {
+      librariesMap[library.importName] = library.code;
+    });
   }
 
   const codeWithWrapper =

diff --git a/test/__tests__/user_transformation.test.js b/test/__tests__/user_transformation.test.js
@@ -1100,6 +1100,67 @@ describe("Timeout tests", () => {
   });
 });
 
+describe("Rudder library tests", () => {
+  beforeEach(() => {});
+  it(`Simple ${name} async test for V1 transformation - with rudder library urlParser `, async () => {
+    const versionId = randomID();
+    const rudderLibraryImportName = '@rs/urlParser/v1';
+    const [name, version] = rudderLibraryImportName.split('/').slice(-2);
+    const inputData = require(`./data/${integration}_input_large.json`);
+    const expectedData = require(`./data/${integration}_async_output_large.json`);
+
+    const respBody = {
+      code: `
+      import url from '@rs/urlParser/v1';
+      async function foo() {
+        return 'resolved';
+      }
+      export async function transformEvent(event, metadata) {
+          const pr = await foo();
+          if(event.properties && event.properties.url){
+            const x = new url.URLSearchParams(event.properties.url).get("client");
+          }
+          event.promise = pr;
+          return event;
+        }
+        `,
+      name: "urlParser",
+      codeVersion: "1"
+    };
+    respBody.versionId = versionId;
+    const transformerUrl = `https://api.rudderlabs.com/transformation/getByVersionId?versionId=${versionId}`;
+    when(fetch)
+      .calledWith(transformerUrl)
+      .mockResolvedValue({
+        status: 200,
+        json: jest.fn().mockResolvedValue(respBody)
+      });
+
+    const urlCode = `${fs.readFileSync(
+      path.resolve(__dirname, "../../src/util/url-search-params.min.js"),
+      "utf8"
+    )};
+    export default self;
+    `;
+
+    const rudderLibraryUrl = `https://api.rudderlabs.com/rudderstackTransformationLibraries/${name}?version=${version}`;
+    when(fetch)
+      .calledWith(rudderLibraryUrl)
+      .mockResolvedValue({
+        status: 200,
+        json: jest.fn().mockResolvedValue({ code: urlCode, name: "urlParser", importName: rudderLibraryImportName })
+      });
+
+    const output = await userTransformHandler(inputData, versionId, []);
+
+    expect(fetch).toHaveBeenCalledWith(
+      `https://api.rudderlabs.com/transformation/getByVersionId?versionId=${versionId}`
+    );
+
+    expect(output).toEqual(expectedData);
+  });
+});
+
 // Running tests for python transformations with openfaas mocks
 describe("Python transformations", () => {
   beforeEach(() => {