chore(release): pull main into develop post release v1.46.4

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.46.4](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.3...v1.46.4) (2023-10-20)
+
+
+### Bug Fixes
+
+* **hs:** time stamp and contact not found issue ([391c7cd](https://github.com/rudderlabs/rudder-transformer/commit/391c7cdc5576152576bbd2c6d335a04ce6795833))
+
 ### [1.46.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.2...v1.46.3) (2023-10-20)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.46.3",
+  "version": "1.46.4",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/v0/destinations/hs/HSTransform-v1.js

@@ -34,6 +34,7 @@ const {
   getEmailAndUpdatedProps,
   formatPropertyValueForIdentify,
   getHsSearchId,
+  populateTraits,
 } = require('./util');
 const { JSON_MIME_TYPE } = require('../../util/constant');
 
@@ -52,7 +53,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant');
  */
 const processLegacyIdentify = async (message, destination, propertyMap) => {
   const { Config } = destination;
-  const traits = getFieldValueFromMessage(message, 'traits');
+  let traits = getFieldValueFromMessage(message, 'traits');
   const mappedToDestination = get(message, MappedToDestinationKey);
   const operation = get(message, 'context.hubspotOperation');
   // if mappedToDestination is set true, then add externalId to traits
@@ -80,6 +81,8 @@ const processLegacyIdentify = async (message, destination, propertyMap) => {
       )}/${hsSearchId}`;
       response.method = defaultPatchRequestConfig.requestMethod;
     }
+
+    traits = await populateTraits(propertyMap, traits, destination);
     response.body.JSON = removeUndefinedAndNullValues({ properties: traits });
     response.source = 'rETL';
     response.operation = operation;

src/v0/destinations/hs/HSTransform-v2.js

@@ -41,6 +41,7 @@ const {
   searchContacts,
   getEventAndPropertiesFromConfig,
   getHsSearchId,
+  populateTraits,
 } = require('./util');
 const { JSON_MIME_TYPE } = require('../../util/constant');
 
@@ -69,7 +70,7 @@ const addHsAuthentication = (response, Config) => {
  */
 const processIdentify = async (message, destination, propertyMap) => {
   const { Config } = destination;
-  const traits = getFieldValueFromMessage(message, 'traits');
+  let traits = getFieldValueFromMessage(message, 'traits');
   const mappedToDestination = get(message, MappedToDestinationKey);
   const operation = get(message, 'context.hubspotOperation');
   const externalIdObj = getDestinationExternalIDObjectForRetl(message, 'HS');
@@ -124,6 +125,7 @@ const processIdentify = async (message, destination, propertyMap) => {
       response.method = defaultPatchRequestConfig.requestMethod;
     }
 
+    traits = await populateTraits(propertyMap, traits, destination);
     response.body.JSON = removeUndefinedAndNullValues({ properties: traits });
     response.source = 'rETL';
     response.operation = operation;

src/v0/destinations/hs/transform.js

@@ -88,6 +88,7 @@ const processRouterDest = async (inputs, reqMetadata) => {
     if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) {
       // skip splitting the batches to inserts and updates if object it is an association
       if (objectType.toLowerCase() !== 'association') {
+        propertyMap = await getProperties(destination);
         // get info about existing objects and splitting accordingly.
         tempInputs = await splitEventsForCreateUpdate(tempInputs, destination);
       }

src/v0/destinations/hs/util.js

@@ -158,23 +158,33 @@ const validatePayloadDataTypes = (propertyMap, hsSupportedKey, value, traitsKey)
 
   if (propertyMap[hsSupportedKey] === 'bool' && typeof propValue === 'object') {
     throw new InstrumentationError(
-      `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${
-        propertyMap[hsSupportedKey]
+      `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${propertyMap[hsSupportedKey]
       }`,
     );
   }
 
   if (propertyMap[hsSupportedKey] === 'number' && typeof propValue !== 'number') {
     throw new InstrumentationError(
-      `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${
-        propertyMap[hsSupportedKey]
+      `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${propertyMap[hsSupportedKey]
       }`,
     );
   }
 
   return propValue;
 };
 
+/**
+ * Converts date to UTC Midnight TimeStamp
+ * @param {*} propValue 
+ * @returns 
+ */
+const getUTCMidnightTimeStampValue = (propValue) => {
+  const time = propValue;
+  const date = new Date(time);
+  date.setUTCHours(0, 0, 0, 0);
+  return date.getTime();
+}
+
 /**
  * add addtional properties in the payload that is provided in traits
  * only when it matches with HS properties (pre-defined/created from dashboard)
@@ -204,10 +214,7 @@ const getTransformedJSON = async (message, destination, propertyMap) => {
       if (!rawPayload[traitsKey] && propertyMap[hsSupportedKey]) {
         let propValue = traits[traitsKey];
         if (propertyMap[hsSupportedKey] === 'date') {
-          const time = propValue;
-          const date = new Date(time);
-          date.setUTCHours(0, 0, 0, 0);
-          propValue = date.getTime();
+          propValue = getUTCMidnightTimeStampValue(propValue);
         }
 
         rawPayload[hsSupportedKey] = validatePayloadDataTypes(
@@ -459,7 +466,7 @@ const getEventAndPropertiesFromConfig = (message, destination, payload) => {
  */
 const getExistingData = async (inputs, destination) => {
   const { Config } = destination;
-  const values = [];
+  let values = [];
   let searchResponse;
   let updateHubspotIds = [];
   const firstMessage = inputs[0].message;
@@ -478,8 +485,10 @@ const getExistingData = async (inputs, destination) => {
   inputs.map(async (input) => {
     const { message } = input;
     const { destinationExternalId } = getDestinationExternalIDInfoForRetl(message, DESTINATION);
-    values.push(destinationExternalId);
+    values.push(destinationExternalId.toString().toLowerCase());
   });
+
+  values = Array.from(new Set(values));
   const requestData = {
     filterGroups: [
       {
@@ -523,15 +532,15 @@ const getExistingData = async (inputs, destination) => {
     searchResponse =
       Config.authorizationType === 'newPrivateAppApi'
         ? await httpPOST(url, requestData, requestOptions, {
-            destType: 'hs',
-            feature: 'transformation',
-            endpointPath,
-          })
+          destType: 'hs',
+          feature: 'transformation',
+          endpointPath,
+        })
         : await httpPOST(url, requestData, {
-            destType: 'hs',
-            feature: 'transformation',
-            endpointPath,
-          });
+          destType: 'hs',
+          feature: 'transformation',
+          endpointPath,
+        });
     searchResponse = processAxiosResponse(searchResponse);
 
     if (searchResponse.status !== 200) {
@@ -626,6 +635,31 @@ const getHsSearchId = (message) => {
   return { hsSearchId };
 };
 
+/**
+ * returns updated traits
+ * @param {*} propertyMap 
+ * @param {*} traits 
+ * @param {*} destination 
+ */
+const populateTraits = async (propertyMap, traits, destination) => {
+  const populatedTraits = traits;
+  let propertyToTypeMap = propertyMap;
+  if (!propertyToTypeMap) {
+    // fetch HS properties
+    propertyToTypeMap = await getProperties(destination);
+  }
+
+  const keys = Object.keys(populatedTraits);
+  keys.forEach((key) => {
+    const value = populatedTraits[key];
+    if (propertyToTypeMap[key] === 'date') {
+      populatedTraits[key] = getUTCMidnightTimeStampValue(value);
+    }
+  })
+
+  return populatedTraits;
+}
+
 module.exports = {
   validateDestinationConfig,
   formatKey,
@@ -639,4 +673,6 @@ module.exports = {
   splitEventsForCreateUpdate,
   getHsSearchId,
   validatePayloadDataTypes,
+  getUTCMidnightTimeStampValue,
+  populateTraits,
 };

test/__mocks__/data/hs/response.json

@@ -3,6 +3,18 @@
     { "name": "company_size", "type": "string" },
     { "name": "date_of_birth", "type": "string" },
     { "name": "days_to_close", "type": "number" },
+    {
+      "name": "date_submitted",
+      "type": "date"
+    },
+    {
+      "name": "days_create",
+      "type": "date"
+    },
+    {
+      "name": "days_closed",
+      "type": "date"
+    },
     { "name": "degree", "type": "string" },
     { "name": "field_of_study", "type": "string" },
     { "name": "first_conversion_date", "type": "datetime" },
@@ -192,6 +204,18 @@
     { "name": "company_size", "type": "string" },
     { "name": "date_of_birth", "type": "string" },
     { "name": "days_to_close", "type": "number" },
+    {
+      "name": "date_submitted",
+      "type": "date"
+    },
+    {
+      "name": "date_created",
+      "type": "date"
+    },
+    {
+      "name": "date_closed",
+      "type": "date"
+    },
     { "name": "degree", "type": "string" },
     { "name": "field_of_study", "type": "string" },
     { "name": "first_conversion_date", "type": "datetime" },

test/__tests__/data/hs_router_rETL_input.json

@@ -216,5 +216,117 @@
     "metadata": {
       "jobId": 3
     }
+  },
+  {
+    "message": {
+      "channel": "web",
+      "context": {
+        "mappedToDestination": true,
+        "externalId": [
+          {
+            "identifierType": "email",
+            "id": "[email protected]",
+            "type": "HS-lead"
+          }
+        ],
+        "sources": {
+          "job_id": "24c5HJxHomh6YCngEOCgjS5r1KX/Syncher",
+          "task_id": "vw_rs_mailchimp_mocked_hg_data",
+          "version": "v1.8.1",
+          "batch_id": "f252c69d-c40d-450e-bcd2-2cf26cb62762",
+          "job_run_id": "c8el40l6e87v0c4hkbl0",
+          "task_run_id": "c8el40l6e87v0c4hkblg"
+        }
+      },
+      "type": "identify",
+      "traits": {
+        "firstname": "Test Hubspot",
+        "anonymousId": "123451",
+        "country": "India",
+        "date_submitted": "2023-09-25T17:31:04.128251Z",
+        "date_created": "2023-03-30T01:02:03.05Z",
+        "date_closed": "2023-10-18T04:38:59.229347Z"
+      },
+      "messageId": "50360b9c-ea8d-409c-b672-c9230f91cce5",
+      "originalTimestamp": "2019-10-15T09:35:31.288Z",
+      "anonymousId": "00000000000000000000000000",
+      "userId": "12345",
+      "integrations": {
+        "All": true
+      },
+      "sentAt": "2019-10-14T09:03:22.563Z"
+    },
+    "destination": {
+      "Config": {
+        "authorizationType": "newPrivateAppApi",
+        "accessToken": "dummy-access-token",
+        "hubID": "dummy-hubId",
+        "apiKey": "dummy-apikey",
+        "apiVersion": "newApi",
+        "lookupField": "lookupField",
+        "hubspotEvents": [
+          {
+            "rsEventName": "Purchase",
+            "hubspotEventName": "pedummy-hubId_rs_hub_test",
+            "eventProperties": [
+              {
+                "from": "Revenue",
+                "to": "value"
+              },
+              {
+                "from": "Price",
+                "to": "cost"
+              }
+            ]
+          },
+          {
+            "rsEventName": "Order Complete",
+            "hubspotEventName": "pedummy-hubId_rs_hub_chair",
+            "eventProperties": [
+              {
+                "from": "firstName",
+                "to": "first_name"
+              },
+              {
+                "from": "lastName",
+                "to": "last_name"
+              }
+            ]
+          }
+        ],
+        "eventFilteringOption": "disable",
+        "blacklistedEvents": [
+          {
+            "eventName": ""
+          }
+        ],
+        "whitelistedEvents": [
+          {
+            "eventName": ""
+          }
+        ]
+      },
+      "secretConfig": {},
+      "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe",
+      "name": "Hubspot",
+      "enabled": true,
+      "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmLd",
+      "deleted": false,
+      "createdAt": "2020-12-30T08:39:32.005Z",
+      "updatedAt": "2021-02-03T16:22:31.374Z",
+      "destinationDefinition": {
+        "id": "1aIXqM806xAVm92nx07YwKbRrO9",
+        "name": "HS",
+        "displayName": "Hubspot",
+        "createdAt": "2020-04-09T09:24:31.794Z",
+        "updatedAt": "2021-01-11T11:03:28.103Z"
+      },
+      "transformations": [],
+      "isConnectionEnabled": true,
+      "isProcessorEnabled": true
+    },
+    "metadata": {
+      "jobId": 4
+    }
   }
 ]