From c50310d1fe372b016c29589b55c3b51a7bdd517d Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Fri, 22 Sep 2023 17:22:26 +0530 Subject: [PATCH 01/15] feat: onboard msl changes for new record event --- .../marketo_static_list/transform.js | 53 ++++++++++++++++--- .../marketo_static_list/processor/data.ts | 8 +-- .../marketo_static_list/router/data.ts | 8 +-- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 7175b5241f..dda5a0caf2 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -60,7 +60,7 @@ const batchResponseBuilder = (message, Config, token, leadIds, operation) => { const processEvent = (input) => { const { token, message, destination } = input; const { Config } = destination; - validateMessageType(message, ['audiencelist']); + validateMessageType(message, ['audiencelist', 'record']); const response = []; let toAdd; let toRemove; @@ -74,14 +74,14 @@ const processEvent = (input) => { (Array.isArray(toAdd) && toAdd.length > 0) || (Array.isArray(toRemove) && toRemove.length > 0) ) { - if (Array.isArray(toAdd) && toAdd.length > 0) { - const payload = batchResponseBuilder(message, Config, token, toAdd, 'add'); + if (Array.isArray(toRemove) && toRemove.length > 0) { + const payload = batchResponseBuilder(message, Config, token, toRemove, 'remove'); if (payload) { response.push(...payload); } } - if (Array.isArray(toRemove) && toRemove.length > 0) { - const payload = batchResponseBuilder(message, Config, token, toRemove, 'remove'); + if (Array.isArray(toAdd) && toAdd.length > 0) { + const payload = batchResponseBuilder(message, Config, token, toAdd, 'add'); if (payload) { response.push(...payload); } @@ -93,6 +93,36 @@ const processEvent = (input) => { } return response; }; + +function transformForRecordEvent(inputs, leadIdObj) { + const tokenisedInputs = inputs.map((input) => { + const { message } = input; + const { fields, action } = message; + if (!message.properties) { + message.properties = {}; + } + if (!message.properties.listData) { + message.properties.listData = { add: [], remove: [] }; + } + // message.properties.listData = message.properties.listData || { add: [], remove: [] }; + if (action === 'insert') { + leadIdObj.insert.push({ id: fields.id }); + message.properties.listData.add.push({ id: fields.id }); + } else if (action === 'delete') { + leadIdObj.delete.push({ id: fields.id }); + message.properties.listData.remove.push({ id: fields.id }); + } else { + throw new InstrumentationError('Invalid action type'); + } + return input; + }); + const finalInput = [tokenisedInputs[0]]; + finalInput[0].message.properties.listData.add = leadIdObj.insert; + finalInput[0].message.properties.listData.remove = leadIdObj.delete; + + return finalInput; +} + const process = async (event) => { const token = await getAuthToken(formatConfig(event.destination)); @@ -107,6 +137,7 @@ const processRouterDest = async (inputs, reqMetadata) => { // Token needs to be generated for marketo which will be done on input level. // If destination information is not present Error should be thrown let token; + let respList; try { token = await getAuthToken(formatConfig(inputs[0].destination)); if (!token) { @@ -140,7 +171,17 @@ const processRouterDest = async (inputs, reqMetadata) => { // If true then previous status is 500 and every subsequent event output should be // sent with status code 500 to the router to be retried. const tokenisedInputs = inputs.map((input) => ({ ...input, token })); - const respList = await simpleProcessRouterDest(tokenisedInputs, processEvent, reqMetadata); + const leadIdObj = { + insert: [], + delete: [], + }; + let finalInputForRecordEvent; + if (inputs[0].message.channel === 'sources') { + finalInputForRecordEvent = transformForRecordEvent(tokenisedInputs, leadIdObj); + respList = await simpleProcessRouterDest(finalInputForRecordEvent, processEvent, reqMetadata); + } else { + respList = await simpleProcessRouterDest(tokenisedInputs, processEvent, reqMetadata); + } return respList; }; diff --git a/test/integrations/destinations/marketo_static_list/processor/data.ts b/test/integrations/destinations/marketo_static_list/processor/data.ts index 51e4e87824..05fba54f6a 100644 --- a/test/integrations/destinations/marketo_static_list/processor/data.ts +++ b/test/integrations/destinations/marketo_static_list/processor/data.ts @@ -62,9 +62,9 @@ export const data = [ output: { version: '1', type: 'REST', - method: 'POST', + method: 'DELETE', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', @@ -80,9 +80,9 @@ export const data = [ output: { version: '1', type: 'REST', - method: 'DELETE', + method: 'POST', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', diff --git a/test/integrations/destinations/marketo_static_list/router/data.ts b/test/integrations/destinations/marketo_static_list/router/data.ts index 23b2ea8ea3..e0d853e1f2 100644 --- a/test/integrations/destinations/marketo_static_list/router/data.ts +++ b/test/integrations/destinations/marketo_static_list/router/data.ts @@ -1232,9 +1232,9 @@ export const data = [ { version: '1', type: 'REST', - method: 'POST', + method: 'DELETE', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', @@ -1251,9 +1251,9 @@ export const data = [ { version: '1', type: 'REST', - method: 'DELETE', + method: 'POST', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', From 2def29e5e5160977a9776d3cd74064fa8d1f0c65 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:26:07 +0530 Subject: [PATCH 02/15] chore: remove older tc for MSL --- test/__tests__/data/marketo_static_list.json | 2542 ----------------- .../marketo_static_list_router_input.json | 1209 -------- ...eto_static_list_router_metadata_input.json | 69 - ...to_static_list_router_metadata_output.json | 20 - .../marketo_static_list_router_output.json | 170 -- test/__tests__/marketo_static_list.test.js | 64 - 6 files changed, 4074 deletions(-) delete mode 100644 test/__tests__/data/marketo_static_list.json delete mode 100644 test/__tests__/data/marketo_static_list_router_input.json delete mode 100644 test/__tests__/data/marketo_static_list_router_metadata_input.json delete mode 100644 test/__tests__/data/marketo_static_list_router_metadata_output.json delete mode 100644 test/__tests__/data/marketo_static_list_router_output.json delete mode 100644 test/__tests__/marketo_static_list.test.js diff --git a/test/__tests__/data/marketo_static_list.json b/test/__tests__/data/marketo_static_list.json deleted file mode 100644 index 701868cab0..0000000000 --- a/test/__tests__/data/marketo_static_list.json +++ /dev/null @@ -1,2542 +0,0 @@ -[ - { - "description": "adding and removing users and getting staticListId from externalId", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 3421 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - } - ], - "remove": [ - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - }, - "externalId": [ - { - "type": "marketoStaticListId", - "id": 1234 - } - ] - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - }, - { - "description": "adding more than max limit users", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 0 - }, - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - }, - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - }, - { - "id": 7 - }, - { - "id": 8 - }, - { - "id": 9 - }, - { - "id": 10 - }, - { - "id": 11 - }, - { - "id": 12 - }, - { - "id": 13 - }, - { - "id": 14 - }, - { - "id": 15 - }, - { - "id": 16 - }, - { - "id": 17 - }, - { - "id": 18 - }, - { - "id": 19 - }, - { - "id": 20 - }, - { - "id": 21 - }, - { - "id": 22 - }, - { - "id": 23 - }, - { - "id": 24 - }, - { - "id": 25 - }, - { - "id": 26 - }, - { - "id": 27 - }, - { - "id": 28 - }, - { - "id": 29 - }, - { - "id": 30 - }, - { - "id": 31 - }, - { - "id": 32 - }, - { - "id": 33 - }, - { - "id": 34 - }, - { - "id": 35 - }, - { - "id": 36 - }, - { - "id": 37 - }, - { - "id": 38 - }, - { - "id": 39 - }, - { - "id": 40 - }, - { - "id": 41 - }, - { - "id": 42 - }, - { - "id": 43 - }, - { - "id": 44 - }, - { - "id": 45 - }, - { - "id": 46 - }, - { - "id": 47 - }, - { - "id": 48 - }, - { - "id": 49 - }, - { - "id": 50 - }, - { - "id": 51 - }, - { - "id": 52 - }, - { - "id": 53 - }, - { - "id": 54 - }, - { - "id": 55 - }, - { - "id": 56 - }, - { - "id": 57 - }, - { - "id": 58 - }, - { - "id": 59 - }, - { - "id": 60 - }, - { - "id": 61 - }, - { - "id": 62 - }, - { - "id": 63 - }, - { - "id": 64 - }, - { - "id": 65 - }, - { - "id": 66 - }, - { - "id": 67 - }, - { - "id": 68 - }, - { - "id": 69 - }, - { - "id": 70 - }, - { - "id": 71 - }, - { - "id": 72 - }, - { - "id": 73 - }, - { - "id": 74 - }, - { - "id": 75 - }, - { - "id": 76 - }, - { - "id": 77 - }, - { - "id": 78 - }, - { - "id": 79 - }, - { - "id": 80 - }, - { - "id": 81 - }, - { - "id": 82 - }, - { - "id": 83 - }, - { - "id": 84 - }, - { - "id": 85 - }, - { - "id": 86 - }, - { - "id": 87 - }, - { - "id": 88 - }, - { - "id": 89 - }, - { - "id": 90 - }, - { - "id": 91 - }, - { - "id": 92 - }, - { - "id": 93 - }, - { - "id": 94 - }, - { - "id": 95 - }, - { - "id": 96 - }, - { - "id": 97 - }, - { - "id": 98 - }, - { - "id": 99 - }, - { - "id": 100 - }, - { - "id": 101 - }, - { - "id": 102 - }, - { - "id": 103 - }, - { - "id": 104 - }, - { - "id": 105 - }, - { - "id": 106 - }, - { - "id": 107 - }, - { - "id": 108 - }, - { - "id": 109 - }, - { - "id": 110 - }, - { - "id": 111 - }, - { - "id": 112 - }, - { - "id": 113 - }, - { - "id": 114 - }, - { - "id": 115 - }, - { - "id": 116 - }, - { - "id": 117 - }, - { - "id": 118 - }, - { - "id": 119 - }, - { - "id": 120 - }, - { - "id": 121 - }, - { - "id": 122 - }, - { - "id": 123 - }, - { - "id": 124 - }, - { - "id": 125 - }, - { - "id": 126 - }, - { - "id": 127 - }, - { - "id": 128 - }, - { - "id": 129 - }, - { - "id": 130 - }, - { - "id": 131 - }, - { - "id": 132 - }, - { - "id": 133 - }, - { - "id": 134 - }, - { - "id": 135 - }, - { - "id": 136 - }, - { - "id": 137 - }, - { - "id": 138 - }, - { - "id": 139 - }, - { - "id": 140 - }, - { - "id": 141 - }, - { - "id": 142 - }, - { - "id": 143 - }, - { - "id": 144 - }, - { - "id": 145 - }, - { - "id": 146 - }, - { - "id": 147 - }, - { - "id": 148 - }, - { - "id": 149 - }, - { - "id": 150 - }, - { - "id": 151 - }, - { - "id": 152 - }, - { - "id": 153 - }, - { - "id": 154 - }, - { - "id": 155 - }, - { - "id": 156 - }, - { - "id": 157 - }, - { - "id": 158 - }, - { - "id": 159 - }, - { - "id": 160 - }, - { - "id": 161 - }, - { - "id": 162 - }, - { - "id": 163 - }, - { - "id": 164 - }, - { - "id": 165 - }, - { - "id": 166 - }, - { - "id": 167 - }, - { - "id": 168 - }, - { - "id": 169 - }, - { - "id": 170 - }, - { - "id": 171 - }, - { - "id": 172 - }, - { - "id": 173 - }, - { - "id": 174 - }, - { - "id": 175 - }, - { - "id": 176 - }, - { - "id": 177 - }, - { - "id": 178 - }, - { - "id": 179 - }, - { - "id": 180 - }, - { - "id": 181 - }, - { - "id": 182 - }, - { - "id": 183 - }, - { - "id": 184 - }, - { - "id": 185 - }, - { - "id": 186 - }, - { - "id": 187 - }, - { - "id": 188 - }, - { - "id": 189 - }, - { - "id": 190 - }, - { - "id": 191 - }, - { - "id": 192 - }, - { - "id": 193 - }, - { - "id": 194 - }, - { - "id": 195 - }, - { - "id": 196 - }, - { - "id": 197 - }, - { - "id": 198 - }, - { - "id": 199 - }, - { - "id": 200 - }, - { - "id": 201 - }, - { - "id": 202 - }, - { - "id": 203 - }, - { - "id": 204 - }, - { - "id": 205 - }, - { - "id": 206 - }, - { - "id": 207 - }, - { - "id": 208 - }, - { - "id": 209 - }, - { - "id": 210 - }, - { - "id": 211 - }, - { - "id": 212 - }, - { - "id": 213 - }, - { - "id": 214 - }, - { - "id": 215 - }, - { - "id": 216 - }, - { - "id": 217 - }, - { - "id": 218 - }, - { - "id": 219 - }, - { - "id": 220 - }, - { - "id": 221 - }, - { - "id": 222 - }, - { - "id": 223 - }, - { - "id": 224 - }, - { - "id": 225 - }, - { - "id": 226 - }, - { - "id": 227 - }, - { - "id": 228 - }, - { - "id": 229 - }, - { - "id": 230 - }, - { - "id": 231 - }, - { - "id": 232 - }, - { - "id": 233 - }, - { - "id": 234 - }, - { - "id": 235 - }, - { - "id": 236 - }, - { - "id": 237 - }, - { - "id": 238 - }, - { - "id": 239 - }, - { - "id": 240 - }, - { - "id": 241 - }, - { - "id": 242 - }, - { - "id": 243 - }, - { - "id": 244 - }, - { - "id": 245 - }, - { - "id": 246 - }, - { - "id": 247 - }, - { - "id": 248 - }, - { - "id": 249 - }, - { - "id": 250 - }, - { - "id": 251 - }, - { - "id": 252 - }, - { - "id": 253 - }, - { - "id": 254 - }, - { - "id": 255 - }, - { - "id": 256 - }, - { - "id": 257 - }, - { - "id": 258 - }, - { - "id": 259 - }, - { - "id": 260 - }, - { - "id": 261 - }, - { - "id": 262 - }, - { - "id": 263 - }, - { - "id": 264 - }, - { - "id": 265 - }, - { - "id": 266 - }, - { - "id": 267 - }, - { - "id": 268 - }, - { - "id": 269 - }, - { - "id": 270 - }, - { - "id": 271 - }, - { - "id": 272 - }, - { - "id": 273 - }, - { - "id": 274 - }, - { - "id": 275 - }, - { - "id": 276 - }, - { - "id": 277 - }, - { - "id": 278 - }, - { - "id": 279 - }, - { - "id": 280 - }, - { - "id": 281 - }, - { - "id": 282 - }, - { - "id": 283 - }, - { - "id": 284 - }, - { - "id": 285 - }, - { - "id": 286 - }, - { - "id": 287 - }, - { - "id": 288 - }, - { - "id": 289 - }, - { - "id": 290 - }, - { - "id": 291 - }, - { - "id": 292 - }, - { - "id": 293 - }, - { - "id": 294 - }, - { - "id": 295 - }, - { - "id": 296 - }, - { - "id": 297 - }, - { - "id": 298 - }, - { - "id": 299 - }, - { - "id": 300 - }, - { - "id": 301 - }, - { - "id": 302 - }, - { - "id": 303 - }, - { - "id": 304 - }, - { - "id": 305 - }, - { - "id": 306 - }, - { - "id": 307 - }, - { - "id": 308 - }, - { - "id": 309 - }, - { - "id": 310 - }, - { - "id": 311 - }, - { - "id": 312 - }, - { - "id": 313 - }, - { - "id": 314 - }, - { - "id": 315 - }, - { - "id": 316 - }, - { - "id": 317 - }, - { - "id": 318 - }, - { - "id": 319 - }, - { - "id": 320 - }, - { - "id": 321 - }, - { - "id": 322 - }, - { - "id": 323 - }, - { - "id": 324 - }, - { - "id": 325 - }, - { - "id": 326 - }, - { - "id": 327 - }, - { - "id": 328 - }, - { - "id": 329 - }, - { - "id": 330 - }, - { - "id": 331 - }, - { - "id": 332 - }, - { - "id": 333 - }, - { - "id": 334 - }, - { - "id": 335 - }, - { - "id": 336 - }, - { - "id": 337 - }, - { - "id": 338 - }, - { - "id": 339 - }, - { - "id": 340 - }, - { - "id": 341 - }, - { - "id": 342 - }, - { - "id": 343 - }, - { - "id": 344 - }, - { - "id": 345 - }, - { - "id": 346 - }, - { - "id": 347 - }, - { - "id": 348 - }, - { - "id": 349 - }, - { - "id": 350 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - }, - { - "description": "removing more than max limit users", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "remove": [ - { - "id": 0 - }, - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - }, - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - }, - { - "id": 7 - }, - { - "id": 8 - }, - { - "id": 9 - }, - { - "id": 10 - }, - { - "id": 11 - }, - { - "id": 12 - }, - { - "id": 13 - }, - { - "id": 14 - }, - { - "id": 15 - }, - { - "id": 16 - }, - { - "id": 17 - }, - { - "id": 18 - }, - { - "id": 19 - }, - { - "id": 20 - }, - { - "id": 21 - }, - { - "id": 22 - }, - { - "id": 23 - }, - { - "id": 24 - }, - { - "id": 25 - }, - { - "id": 26 - }, - { - "id": 27 - }, - { - "id": 28 - }, - { - "id": 29 - }, - { - "id": 30 - }, - { - "id": 31 - }, - { - "id": 32 - }, - { - "id": 33 - }, - { - "id": 34 - }, - { - "id": 35 - }, - { - "id": 36 - }, - { - "id": 37 - }, - { - "id": 38 - }, - { - "id": 39 - }, - { - "id": 40 - }, - { - "id": 41 - }, - { - "id": 42 - }, - { - "id": 43 - }, - { - "id": 44 - }, - { - "id": 45 - }, - { - "id": 46 - }, - { - "id": 47 - }, - { - "id": 48 - }, - { - "id": 49 - }, - { - "id": 50 - }, - { - "id": 51 - }, - { - "id": 52 - }, - { - "id": 53 - }, - { - "id": 54 - }, - { - "id": 55 - }, - { - "id": 56 - }, - { - "id": 57 - }, - { - "id": 58 - }, - { - "id": 59 - }, - { - "id": 60 - }, - { - "id": 61 - }, - { - "id": 62 - }, - { - "id": 63 - }, - { - "id": 64 - }, - { - "id": 65 - }, - { - "id": 66 - }, - { - "id": 67 - }, - { - "id": 68 - }, - { - "id": 69 - }, - { - "id": 70 - }, - { - "id": 71 - }, - { - "id": 72 - }, - { - "id": 73 - }, - { - "id": 74 - }, - { - "id": 75 - }, - { - "id": 76 - }, - { - "id": 77 - }, - { - "id": 78 - }, - { - "id": 79 - }, - { - "id": 80 - }, - { - "id": 81 - }, - { - "id": 82 - }, - { - "id": 83 - }, - { - "id": 84 - }, - { - "id": 85 - }, - { - "id": 86 - }, - { - "id": 87 - }, - { - "id": 88 - }, - { - "id": 89 - }, - { - "id": 90 - }, - { - "id": 91 - }, - { - "id": 92 - }, - { - "id": 93 - }, - { - "id": 94 - }, - { - "id": 95 - }, - { - "id": 96 - }, - { - "id": 97 - }, - { - "id": 98 - }, - { - "id": 99 - }, - { - "id": 100 - }, - { - "id": 101 - }, - { - "id": 102 - }, - { - "id": 103 - }, - { - "id": 104 - }, - { - "id": 105 - }, - { - "id": 106 - }, - { - "id": 107 - }, - { - "id": 108 - }, - { - "id": 109 - }, - { - "id": 110 - }, - { - "id": 111 - }, - { - "id": 112 - }, - { - "id": 113 - }, - { - "id": 114 - }, - { - "id": 115 - }, - { - "id": 116 - }, - { - "id": 117 - }, - { - "id": 118 - }, - { - "id": 119 - }, - { - "id": 120 - }, - { - "id": 121 - }, - { - "id": 122 - }, - { - "id": 123 - }, - { - "id": 124 - }, - { - "id": 125 - }, - { - "id": 126 - }, - { - "id": 127 - }, - { - "id": 128 - }, - { - "id": 129 - }, - { - "id": 130 - }, - { - "id": 131 - }, - { - "id": 132 - }, - { - "id": 133 - }, - { - "id": 134 - }, - { - "id": 135 - }, - { - "id": 136 - }, - { - "id": 137 - }, - { - "id": 138 - }, - { - "id": 139 - }, - { - "id": 140 - }, - { - "id": 141 - }, - { - "id": 142 - }, - { - "id": 143 - }, - { - "id": 144 - }, - { - "id": 145 - }, - { - "id": 146 - }, - { - "id": 147 - }, - { - "id": 148 - }, - { - "id": 149 - }, - { - "id": 150 - }, - { - "id": 151 - }, - { - "id": 152 - }, - { - "id": 153 - }, - { - "id": 154 - }, - { - "id": 155 - }, - { - "id": 156 - }, - { - "id": 157 - }, - { - "id": 158 - }, - { - "id": 159 - }, - { - "id": 160 - }, - { - "id": 161 - }, - { - "id": 162 - }, - { - "id": 163 - }, - { - "id": 164 - }, - { - "id": 165 - }, - { - "id": 166 - }, - { - "id": 167 - }, - { - "id": 168 - }, - { - "id": 169 - }, - { - "id": 170 - }, - { - "id": 171 - }, - { - "id": 172 - }, - { - "id": 173 - }, - { - "id": 174 - }, - { - "id": 175 - }, - { - "id": 176 - }, - { - "id": 177 - }, - { - "id": 178 - }, - { - "id": 179 - }, - { - "id": 180 - }, - { - "id": 181 - }, - { - "id": 182 - }, - { - "id": 183 - }, - { - "id": 184 - }, - { - "id": 185 - }, - { - "id": 186 - }, - { - "id": 187 - }, - { - "id": 188 - }, - { - "id": 189 - }, - { - "id": 190 - }, - { - "id": 191 - }, - { - "id": 192 - }, - { - "id": 193 - }, - { - "id": 194 - }, - { - "id": 195 - }, - { - "id": 196 - }, - { - "id": 197 - }, - { - "id": 198 - }, - { - "id": 199 - }, - { - "id": 200 - }, - { - "id": 201 - }, - { - "id": 202 - }, - { - "id": 203 - }, - { - "id": 204 - }, - { - "id": 205 - }, - { - "id": 206 - }, - { - "id": 207 - }, - { - "id": 208 - }, - { - "id": 209 - }, - { - "id": 210 - }, - { - "id": 211 - }, - { - "id": 212 - }, - { - "id": 213 - }, - { - "id": 214 - }, - { - "id": 215 - }, - { - "id": 216 - }, - { - "id": 217 - }, - { - "id": 218 - }, - { - "id": 219 - }, - { - "id": 220 - }, - { - "id": 221 - }, - { - "id": 222 - }, - { - "id": 223 - }, - { - "id": 224 - }, - { - "id": 225 - }, - { - "id": 226 - }, - { - "id": 227 - }, - { - "id": 228 - }, - { - "id": 229 - }, - { - "id": 230 - }, - { - "id": 231 - }, - { - "id": 232 - }, - { - "id": 233 - }, - { - "id": 234 - }, - { - "id": 235 - }, - { - "id": 236 - }, - { - "id": 237 - }, - { - "id": 238 - }, - { - "id": 239 - }, - { - "id": 240 - }, - { - "id": 241 - }, - { - "id": 242 - }, - { - "id": 243 - }, - { - "id": 244 - }, - { - "id": 245 - }, - { - "id": 246 - }, - { - "id": 247 - }, - { - "id": 248 - }, - { - "id": 249 - }, - { - "id": 250 - }, - { - "id": 251 - }, - { - "id": 252 - }, - { - "id": 253 - }, - { - "id": 254 - }, - { - "id": 255 - }, - { - "id": 256 - }, - { - "id": 257 - }, - { - "id": 258 - }, - { - "id": 259 - }, - { - "id": 260 - }, - { - "id": 261 - }, - { - "id": 262 - }, - { - "id": 263 - }, - { - "id": 264 - }, - { - "id": 265 - }, - { - "id": 266 - }, - { - "id": 267 - }, - { - "id": 268 - }, - { - "id": 269 - }, - { - "id": 270 - }, - { - "id": 271 - }, - { - "id": 272 - }, - { - "id": 273 - }, - { - "id": 274 - }, - { - "id": 275 - }, - { - "id": 276 - }, - { - "id": 277 - }, - { - "id": 278 - }, - { - "id": 279 - }, - { - "id": 280 - }, - { - "id": 281 - }, - { - "id": 282 - }, - { - "id": 283 - }, - { - "id": 284 - }, - { - "id": 285 - }, - { - "id": 286 - }, - { - "id": 287 - }, - { - "id": 288 - }, - { - "id": 289 - }, - { - "id": 290 - }, - { - "id": 291 - }, - { - "id": 292 - }, - { - "id": 293 - }, - { - "id": 294 - }, - { - "id": 295 - }, - { - "id": 296 - }, - { - "id": 297 - }, - { - "id": 298 - }, - { - "id": 299 - }, - { - "id": 300 - }, - { - "id": 301 - }, - { - "id": 302 - }, - { - "id": 303 - }, - { - "id": 304 - }, - { - "id": 305 - }, - { - "id": 306 - }, - { - "id": 307 - }, - { - "id": 308 - }, - { - "id": 309 - }, - { - "id": 310 - }, - { - "id": 311 - }, - { - "id": 312 - }, - { - "id": 313 - }, - { - "id": 314 - }, - { - "id": 315 - }, - { - "id": 316 - }, - { - "id": 317 - }, - { - "id": 318 - }, - { - "id": 319 - }, - { - "id": 320 - }, - { - "id": 321 - }, - { - "id": 322 - }, - { - "id": 323 - }, - { - "id": 324 - }, - { - "id": 325 - }, - { - "id": 326 - }, - { - "id": 327 - }, - { - "id": 328 - }, - { - "id": 329 - }, - { - "id": 330 - }, - { - "id": 331 - }, - { - "id": 332 - }, - { - "id": 333 - }, - { - "id": 334 - }, - { - "id": 335 - }, - { - "id": 336 - }, - { - "id": 337 - }, - { - "id": 338 - }, - { - "id": 339 - }, - { - "id": 340 - }, - { - "id": 341 - }, - { - "id": 342 - }, - { - "id": 343 - }, - { - "id": 344 - }, - { - "id": 345 - }, - { - "id": 346 - }, - { - "id": 347 - }, - { - "id": 348 - }, - { - "id": 349 - }, - { - "id": 350 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - }, - { - "description": "unsupported message Type", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "track", - "properties": { - "listData": { - "remove": [1] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": { - "error": "Event type track is not supported" - } - }, - { - "description": "Invalid leadIds format ", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "remove": 1 - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": { - "error": "Invalid leadIds format or no leadIds found neither to add nor to remove" - } - }, - { - "description": "Only adding with remove not an array", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [{ "id": 1 }], - "remove": 2 - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - } -] diff --git a/test/__tests__/data/marketo_static_list_router_input.json b/test/__tests__/data/marketo_static_list_router_input.json deleted file mode 100644 index ed3a7591e6..0000000000 --- a/test/__tests__/data/marketo_static_list_router_input.json +++ /dev/null @@ -1,1209 +0,0 @@ -[ - { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - } - ], - "remove": [ - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - }, - "metadata": { - "jobId": 1 - } - }, - { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 0 - }, - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - }, - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - }, - { - "id": 7 - }, - { - "id": 8 - }, - { - "id": 9 - }, - { - "id": 10 - }, - { - "id": 11 - }, - { - "id": 12 - }, - { - "id": 13 - }, - { - "id": 14 - }, - { - "id": 15 - }, - { - "id": 16 - }, - { - "id": 17 - }, - { - "id": 18 - }, - { - "id": 19 - }, - { - "id": 20 - }, - { - "id": 21 - }, - { - "id": 22 - }, - { - "id": 23 - }, - { - "id": 24 - }, - { - "id": 25 - }, - { - "id": 26 - }, - { - "id": 27 - }, - { - "id": 28 - }, - { - "id": 29 - }, - { - "id": 30 - }, - { - "id": 31 - }, - { - "id": 32 - }, - { - "id": 33 - }, - { - "id": 34 - }, - { - "id": 35 - }, - { - "id": 36 - }, - { - "id": 37 - }, - { - "id": 38 - }, - { - "id": 39 - }, - { - "id": 40 - }, - { - "id": 41 - }, - { - "id": 42 - }, - { - "id": 43 - }, - { - "id": 44 - }, - { - "id": 45 - }, - { - "id": 46 - }, - { - "id": 47 - }, - { - "id": 48 - }, - { - "id": 49 - }, - { - "id": 50 - }, - { - "id": 51 - }, - { - "id": 52 - }, - { - "id": 53 - }, - { - "id": 54 - }, - { - "id": 55 - }, - { - "id": 56 - }, - { - "id": 57 - }, - { - "id": 58 - }, - { - "id": 59 - }, - { - "id": 60 - }, - { - "id": 61 - }, - { - "id": 62 - }, - { - "id": 63 - }, - { - "id": 64 - }, - { - "id": 65 - }, - { - "id": 66 - }, - { - "id": 67 - }, - { - "id": 68 - }, - { - "id": 69 - }, - { - "id": 70 - }, - { - "id": 71 - }, - { - "id": 72 - }, - { - "id": 73 - }, - { - "id": 74 - }, - { - "id": 75 - }, - { - "id": 76 - }, - { - "id": 77 - }, - { - "id": 78 - }, - { - "id": 79 - }, - { - "id": 80 - }, - { - "id": 81 - }, - { - "id": 82 - }, - { - "id": 83 - }, - { - "id": 84 - }, - { - "id": 85 - }, - { - "id": 86 - }, - { - "id": 87 - }, - { - "id": 88 - }, - { - "id": 89 - }, - { - "id": 90 - }, - { - "id": 91 - }, - { - "id": 92 - }, - { - "id": 93 - }, - { - "id": 94 - }, - { - "id": 95 - }, - { - "id": 96 - }, - { - "id": 97 - }, - { - "id": 98 - }, - { - "id": 99 - }, - { - "id": 100 - }, - { - "id": 101 - }, - { - "id": 102 - }, - { - "id": 103 - }, - { - "id": 104 - }, - { - "id": 105 - }, - { - "id": 106 - }, - { - "id": 107 - }, - { - "id": 108 - }, - { - "id": 109 - }, - { - "id": 110 - }, - { - "id": 111 - }, - { - "id": 112 - }, - { - "id": 113 - }, - { - "id": 114 - }, - { - "id": 115 - }, - { - "id": 116 - }, - { - "id": 117 - }, - { - "id": 118 - }, - { - "id": 119 - }, - { - "id": 120 - }, - { - "id": 121 - }, - { - "id": 122 - }, - { - "id": 123 - }, - { - "id": 124 - }, - { - "id": 125 - }, - { - "id": 126 - }, - { - "id": 127 - }, - { - "id": 128 - }, - { - "id": 129 - }, - { - "id": 130 - }, - { - "id": 131 - }, - { - "id": 132 - }, - { - "id": 133 - }, - { - "id": 134 - }, - { - "id": 135 - }, - { - "id": 136 - }, - { - "id": 137 - }, - { - "id": 138 - }, - { - "id": 139 - }, - { - "id": 140 - }, - { - "id": 141 - }, - { - "id": 142 - }, - { - "id": 143 - }, - { - "id": 144 - }, - { - "id": 145 - }, - { - "id": 146 - }, - { - "id": 147 - }, - { - "id": 148 - }, - { - "id": 149 - }, - { - "id": 150 - }, - { - "id": 151 - }, - { - "id": 152 - }, - { - "id": 153 - }, - { - "id": 154 - }, - { - "id": 155 - }, - { - "id": 156 - }, - { - "id": 157 - }, - { - "id": 158 - }, - { - "id": 159 - }, - { - "id": 160 - }, - { - "id": 161 - }, - { - "id": 162 - }, - { - "id": 163 - }, - { - "id": 164 - }, - { - "id": 165 - }, - { - "id": 166 - }, - { - "id": 167 - }, - { - "id": 168 - }, - { - "id": 169 - }, - { - "id": 170 - }, - { - "id": 171 - }, - { - "id": 172 - }, - { - "id": 173 - }, - { - "id": 174 - }, - { - "id": 175 - }, - { - "id": 176 - }, - { - "id": 177 - }, - { - "id": 178 - }, - { - "id": 179 - }, - { - "id": 180 - }, - { - "id": 181 - }, - { - "id": 182 - }, - { - "id": 183 - }, - { - "id": 184 - }, - { - "id": 185 - }, - { - "id": 186 - }, - { - "id": 187 - }, - { - "id": 188 - }, - { - "id": 189 - }, - { - "id": 190 - }, - { - "id": 191 - }, - { - "id": 192 - }, - { - "id": 193 - }, - { - "id": 194 - }, - { - "id": 195 - }, - { - "id": 196 - }, - { - "id": 197 - }, - { - "id": 198 - }, - { - "id": 199 - }, - { - "id": 200 - }, - { - "id": 201 - }, - { - "id": 202 - }, - { - "id": 203 - }, - { - "id": 204 - }, - { - "id": 205 - }, - { - "id": 206 - }, - { - "id": 207 - }, - { - "id": 208 - }, - { - "id": 209 - }, - { - "id": 210 - }, - { - "id": 211 - }, - { - "id": 212 - }, - { - "id": 213 - }, - { - "id": 214 - }, - { - "id": 215 - }, - { - "id": 216 - }, - { - "id": 217 - }, - { - "id": 218 - }, - { - "id": 219 - }, - { - "id": 220 - }, - { - "id": 221 - }, - { - "id": 222 - }, - { - "id": 223 - }, - { - "id": 224 - }, - { - "id": 225 - }, - { - "id": 226 - }, - { - "id": 227 - }, - { - "id": 228 - }, - { - "id": 229 - }, - { - "id": 230 - }, - { - "id": 231 - }, - { - "id": 232 - }, - { - "id": 233 - }, - { - "id": 234 - }, - { - "id": 235 - }, - { - "id": 236 - }, - { - "id": 237 - }, - { - "id": 238 - }, - { - "id": 239 - }, - { - "id": 240 - }, - { - "id": 241 - }, - { - "id": 242 - }, - { - "id": 243 - }, - { - "id": 244 - }, - { - "id": 245 - }, - { - "id": 246 - }, - { - "id": 247 - }, - { - "id": 248 - }, - { - "id": 249 - }, - { - "id": 250 - }, - { - "id": 251 - }, - { - "id": 252 - }, - { - "id": 253 - }, - { - "id": 254 - }, - { - "id": 255 - }, - { - "id": 256 - }, - { - "id": 257 - }, - { - "id": 258 - }, - { - "id": 259 - }, - { - "id": 260 - }, - { - "id": 261 - }, - { - "id": 262 - }, - { - "id": 263 - }, - { - "id": 264 - }, - { - "id": 265 - }, - { - "id": 266 - }, - { - "id": 267 - }, - { - "id": 268 - }, - { - "id": 269 - }, - { - "id": 270 - }, - { - "id": 271 - }, - { - "id": 272 - }, - { - "id": 273 - }, - { - "id": 274 - }, - { - "id": 275 - }, - { - "id": 276 - }, - { - "id": 277 - }, - { - "id": 278 - }, - { - "id": 279 - }, - { - "id": 280 - }, - { - "id": 281 - }, - { - "id": 282 - }, - { - "id": 283 - }, - { - "id": 284 - }, - { - "id": 285 - }, - { - "id": 286 - }, - { - "id": 287 - }, - { - "id": 288 - }, - { - "id": 289 - }, - { - "id": 290 - }, - { - "id": 291 - }, - { - "id": 292 - }, - { - "id": 293 - }, - { - "id": 294 - }, - { - "id": 295 - }, - { - "id": 296 - }, - { - "id": 297 - }, - { - "id": 298 - }, - { - "id": 299 - }, - { - "id": 300 - }, - { - "id": 301 - }, - { - "id": 302 - }, - { - "id": 303 - }, - { - "id": 304 - }, - { - "id": 305 - }, - { - "id": 306 - }, - { - "id": 307 - }, - { - "id": 308 - }, - { - "id": 309 - }, - { - "id": 310 - }, - { - "id": 311 - }, - { - "id": 312 - }, - { - "id": 313 - }, - { - "id": 314 - }, - { - "id": 315 - }, - { - "id": 316 - }, - { - "id": 317 - }, - { - "id": 318 - }, - { - "id": 319 - }, - { - "id": 320 - }, - { - "id": 321 - }, - { - "id": 322 - }, - { - "id": 323 - }, - { - "id": 324 - }, - { - "id": 325 - }, - { - "id": 326 - }, - { - "id": 327 - }, - { - "id": 328 - }, - { - "id": 329 - }, - { - "id": 330 - }, - { - "id": 331 - }, - { - "id": 332 - }, - { - "id": 333 - }, - { - "id": 334 - }, - { - "id": 335 - }, - { - "id": 336 - }, - { - "id": 337 - }, - { - "id": 338 - }, - { - "id": 339 - }, - { - "id": 340 - }, - { - "id": 341 - }, - { - "id": 342 - }, - { - "id": 343 - }, - { - "id": 344 - }, - { - "id": 345 - }, - { - "id": 346 - }, - { - "id": 347 - }, - { - "id": 348 - }, - { - "id": 349 - }, - { - "id": 350 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - }, - "metadata": { - "jobId": 2 - } - }, - { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "track", - "properties": { - "listData": { - "remove": [1] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - }, - "metadata": { - "jobId": 3 - } - } -] diff --git a/test/__tests__/data/marketo_static_list_router_metadata_input.json b/test/__tests__/data/marketo_static_list_router_metadata_input.json deleted file mode 100644 index d006aa184b..0000000000 --- a/test/__tests__/data/marketo_static_list_router_metadata_input.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "batchedRequest": { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer access_token_success" - }, - "params": {}, - "body": { - "JSON": {}, - "XML": {}, - "JSON_ARRAY": {}, - "FORM": {} - }, - "files": {} - }, - "metadata": [ - { - "jobId": 1 - }, - { - "jobId": 2 - }, - { - "jobId": 3 - } - ], - "batched": false, - "statusCode": 200, - "destination": { - "Config": { - "accountId": "marketo_acct_id_success", - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "staticListId": 3027 - }, - "secretConfig": {}, - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "name": "Marketo Static List", - "enabled": true, - "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmLd", - "deleted": false, - "createdAt": "2020-12-30T08:39:32.005Z", - "updatedAt": "2021-02-03T16:22:31.374Z", - "destinationDefinition": { - "config": { - "destConfig": { - "defaultConfig": ["accountId", "clientId", "clientSecret", "staticListId"] - }, - "secretKeys": ["clientSecret"], - "excludeKeys": [], - "includeKeys": [], - "routerTransform": true, - "supportedSourceTypes": ["android", "ios", "web", "unity", "amp", "cloud", "reactnative"] - }, - "id": "1aIXqM806xAVm92nx07YwKbRrO9", - "name": "MARKETO_STATIC_LIST", - "displayName": "Marketo Static List", - "createdAt": "2020-04-09T09:24:31.794Z", - "updatedAt": "2021-01-11T11:03:28.103Z" - }, - "transformations": [], - "isConnectionEnabled": true, - "isProcessorEnabled": true - } -} diff --git a/test/__tests__/data/marketo_static_list_router_metadata_output.json b/test/__tests__/data/marketo_static_list_router_metadata_output.json deleted file mode 100644 index bc560d73a7..0000000000 --- a/test/__tests__/data/marketo_static_list_router_metadata_output.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "destInfo": { - "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" - }, - "jobId": 1 - }, - { - "destInfo": { - "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" - }, - "jobId": 2 - }, - { - "destInfo": { - "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" - }, - "jobId": 3 - } -] diff --git a/test/__tests__/data/marketo_static_list_router_output.json b/test/__tests__/data/marketo_static_list_router_output.json deleted file mode 100644 index 9c096b84c6..0000000000 --- a/test/__tests__/data/marketo_static_list_router_output.json +++ /dev/null @@ -1,170 +0,0 @@ -[ - { - "batchedRequest": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ], - "metadata": [ - { - "jobId": 1 - } - ], - "batched": false, - "statusCode": 200, - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - } - }, - { - "batchedRequest": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ], - "metadata": [ - { - "jobId": 2 - } - ], - "batched": false, - "statusCode": 200, - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - } - }, - { - "metadata": [ - { - "jobId": 3 - } - ], - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "batched": false, - "statusCode": 400, - "error": "Event type track is not supported", - "statTags": { - "errorCategory": "dataValidation", - "errorType": "instrumentation" - } - } -] diff --git a/test/__tests__/marketo_static_list.test.js b/test/__tests__/marketo_static_list.test.js deleted file mode 100644 index 43300f243a..0000000000 --- a/test/__tests__/marketo_static_list.test.js +++ /dev/null @@ -1,64 +0,0 @@ -const integration = "marketo_static_list"; -const name = "marketo_static_list"; -const version = "v0"; - -const fs = require("fs"); -const path = require("path"); - -const transformer = require(`../../src/${version}/destinations/${integration}/transform`); - -// Processor Test Data -const testDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}.json`) -); -const testData = JSON.parse(testDataFile); - -// Router Test Data -const inputRouterDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_input.json`) -); -const outputRouterDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_output.json`) -); -const inputRouterData = JSON.parse(inputRouterDataFile); -const expectedRouterData = JSON.parse(outputRouterDataFile); - -// Router Metadata Test files -const inputRouterMetadataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_metadata_input.json`) -); -const outputRouterMetadataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_metadata_output.json`) -); -const inputRouterMetadata = JSON.parse(inputRouterMetadataFile); -const expectedRouterMetadata = JSON.parse(outputRouterMetadataFile); - -describe(`${name} Tests`, () => { - describe("Processor", () => { - testData.forEach((dataPoint, index) => { - it(`${index}. ${integration} - ${dataPoint.description}`, async () => { - try { - const output = await transformer.process(dataPoint.input); - expect(output).toEqual(dataPoint.output); - } catch (error) { - expect(error.message).toEqual(dataPoint.output.error); - } - }); - }); - }); - - describe("Router Tests", () => { - it("Payload", async () => { - const routerOutput = await transformer.processRouterDest(inputRouterData); - expect(routerOutput).toEqual(expectedRouterData); - }); - }); - describe("Router Metadata Tests", () => { - it("Payload", async () => { - const routerMetadataOutput = await transformer.processMetadataForRouter( - inputRouterMetadata - ); - expect(routerMetadataOutput).toEqual(expectedRouterMetadata); - }); - }); -}); From b6d73c40f88a64dc51b7af4a7da41c4700ccd3fb Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:13:29 +0530 Subject: [PATCH 03/15] fix: metadata --- src/v0/destinations/marketo_static_list/transform.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index dda5a0caf2..67323ae7c6 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -95,8 +95,12 @@ const processEvent = (input) => { }; function transformForRecordEvent(inputs, leadIdObj) { + const finalMetadata = []; + // iterate through each inputs metadata and create a final metadata const tokenisedInputs = inputs.map((input) => { const { message } = input; + const { metadata } = input; + finalMetadata.push(metadata); const { fields, action } = message; if (!message.properties) { message.properties = {}; @@ -117,6 +121,7 @@ function transformForRecordEvent(inputs, leadIdObj) { return input; }); const finalInput = [tokenisedInputs[0]]; + finalInput[0].metadata = finalMetadata; finalInput[0].message.properties.listData.add = leadIdObj.insert; finalInput[0].message.properties.listData.remove = leadIdObj.delete; From f46149d413011218551c7f82b66dc41d8ae6199f Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:36:19 +0530 Subject: [PATCH 04/15] fix: metadatax2 --- src/v0/destinations/marketo_static_list/transform.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 67323ae7c6..d4e5c4d500 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -187,6 +187,7 @@ const processRouterDest = async (inputs, reqMetadata) => { } else { respList = await simpleProcessRouterDest(tokenisedInputs, processEvent, reqMetadata); } + // [respList[0].metadata] = respList[0].metadata; return respList; }; @@ -197,7 +198,13 @@ const processRouterDest = async (inputs, reqMetadata) => { */ function processMetadataForRouter(output) { const { metadata, destination } = output; - const clonedMetadata = cloneDeep(metadata); + // check if metadata[0] is an array or not + let clonedMetadata; + if (Array.isArray(metadata[0])) { + [clonedMetadata] = cloneDeep(metadata); + } else { + clonedMetadata = cloneDeep(metadata); + } clonedMetadata.forEach((metadataElement) => { // eslint-disable-next-line no-param-reassign metadataElement.destInfo = { authKey: destination.ID }; From 879bc063beafba98c7bac528527bb17481de3d2f Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:19:15 +0530 Subject: [PATCH 05/15] chore: address comments --- .../marketo_static_list/transform.js | 110 ++++---- .../destinations/marketo_static_list/util.js | 40 +++ .../marketo_static_list/util.test.js | 239 ++++++++++++++++++ 3 files changed, 335 insertions(+), 54 deletions(-) create mode 100644 src/v0/destinations/marketo_static_list/util.test.js diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 976db8d740..1faae70443 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -6,7 +6,7 @@ const { generateErrorObject, } = require('../../util'); const { AUTH_CACHE_TTL, JSON_MIME_TYPE } = require('../../util/constant'); -const { getIds, validateMessageType } = require('./util'); +const { getIds, validateMessageType, transformForRecordEvent } = require('./util'); const { getDestinationExternalID, defaultRequestConfig, @@ -16,7 +16,7 @@ const { const { formatConfig, MAX_LEAD_IDS_SIZE } = require('./config'); const Cache = require('../../util/cache'); const { getAuthToken } = require('../marketo/transform'); -const { InstrumentationError, UnauthorizedError } = require('../../util/errorTypes'); +const { InstrumentationError, UnauthorizedError, AbortedError } = require('../../util/errorTypes'); const authCache = new Cache(AUTH_CACHE_TTL); // 1 hr @@ -71,9 +71,13 @@ const processEvent = (input) => { toRemove = getIds(message.properties.listData.remove); } if ( - (Array.isArray(toAdd) && toAdd.length > 0) || - (Array.isArray(toRemove) && toRemove.length > 0) + (!Array.isArray(toAdd) || toAdd.length === 0) && + (!Array.isArray(toRemove) || toRemove.length === 0) ) { + throw new InstrumentationError( + 'Invalid leadIds format or no leadIds found neither to add nor to remove', + ); + } else { if (Array.isArray(toRemove) && toRemove.length > 0) { const payload = batchResponseBuilder(message, Config, token, toRemove, 'remove'); if (payload) { @@ -86,47 +90,44 @@ const processEvent = (input) => { response.push(...payload); } } - } else { - throw new InstrumentationError( - 'Invalid leadIds format or no leadIds found neither to add nor to remove', - ); } return response; }; -function transformForRecordEvent(inputs, leadIdObj) { - const finalMetadata = []; - // iterate through each inputs metadata and create a final metadata - const tokenisedInputs = inputs.map((input) => { - const { message } = input; - const { metadata } = input; - finalMetadata.push(metadata); - const { fields, action } = message; - if (!message.properties) { - message.properties = {}; - } - if (!message.properties.listData) { - message.properties.listData = { add: [], remove: [] }; - } - // message.properties.listData = message.properties.listData || { add: [], remove: [] }; - if (action === 'insert') { - leadIdObj.insert.push({ id: fields.id }); - message.properties.listData.add.push({ id: fields.id }); - } else if (action === 'delete') { - leadIdObj.delete.push({ id: fields.id }); - message.properties.listData.remove.push({ id: fields.id }); - } else { - throw new InstrumentationError('Invalid action type'); - } - return input; - }); - const finalInput = [tokenisedInputs[0]]; - finalInput[0].metadata = finalMetadata; - finalInput[0].message.properties.listData.add = leadIdObj.insert; - finalInput[0].message.properties.listData.remove = leadIdObj.delete; +// function transformForRecordEvent(inputs, leadIdObj) { +// const finalMetadata = []; +// // iterate through each inputs metadata and create a final metadata +// const tokenisedInputs = inputs.map((input) => { +// const { message } = input; +// const { metadata } = input; +// finalMetadata.push(metadata); +// const { fields, action } = message; +// if (!message.properties) { +// message.properties = {}; +// } +// if (!message.properties.listData) { +// message.properties.listData = { add: [], remove: [] }; +// } +// // message.properties.listData = message.properties.listData || { add: [], remove: [] }; +// const fieldsId = fields.id; +// if (action === 'insert') { +// leadIdObj.insert.push({ id: fieldsId }); +// message.properties.listData.add.push({ id: fieldsId }); +// } else if (action === 'delete') { +// leadIdObj.delete.push({ id: fieldsId }); +// message.properties.listData.remove.push({ id: fieldsId }); +// } else { +// throw new InstrumentationError('Invalid action type'); +// } +// return input; +// }); +// const finalInput = [tokenisedInputs[0]]; +// finalInput[0].metadata = finalMetadata; +// finalInput[0].message.properties.listData.add = leadIdObj.insert; +// finalInput[0].message.properties.listData.remove = leadIdObj.delete; - return finalInput; -} +// return finalInput; +// } const process = async (event) => { const token = await getAuthToken(formatConfig(event.destination)); @@ -146,19 +147,20 @@ const processRouterDest = async (inputs, reqMetadata) => { try { token = await getAuthToken(formatConfig(inputs[0].destination)); if (!token) { - const errResp = { - status: 400, - message: 'Authorisation failed', - responseTransformFailure: true, - statTags: {}, - }; - const respEvents = getErrorRespEvents( - inputs.map((input) => input.metadata), - errResp.status, - errResp.message, - errResp.statTags, - ); - return [{ ...respEvents, destination: inputs?.[0]?.destination }]; + throw new AbortedError('Could not retrieve authorisation token', 400); + // const errResp = { + // status: 400, + // message: 'Authorisation failed', + // responseTransformFailure: true, + // statTags: {}, + // }; + // const respEvents = getErrorRespEvents( + // inputs.map((input) => input.metadata), + // errResp.status, + // errResp.message, + // errResp.statTags, + // ); + // return [{ ...respEvents, destination: inputs?.[0]?.destination }]; } } catch (error) { // Not using handleRtTfSingleEventError here as this is for multiple events @@ -181,7 +183,7 @@ const processRouterDest = async (inputs, reqMetadata) => { delete: [], }; let finalInputForRecordEvent; - if (inputs[0].message.channel === 'sources') { + if (inputs[0].message.channel === 'sources' && inputs[0].message.type === 'record') { finalInputForRecordEvent = transformForRecordEvent(tokenisedInputs, leadIdObj); respList = await simpleProcessRouterDest(finalInputForRecordEvent, processEvent, reqMetadata); } else { diff --git a/src/v0/destinations/marketo_static_list/util.js b/src/v0/destinations/marketo_static_list/util.js index 2d4fe493a0..4427891640 100644 --- a/src/v0/destinations/marketo_static_list/util.js +++ b/src/v0/destinations/marketo_static_list/util.js @@ -34,7 +34,47 @@ const validateMessageType = (message, allowedTypes) => { } }; +function transformForRecordEvent(inputs, leadIdObj) { + const finalMetadata = []; + // iterate through each inputs metadata and create a final metadata + const tokenisedInputs = inputs.map((input) => { + const { message } = input; + const { metadata } = input; + finalMetadata.push(metadata); + const { fields, action } = message; + let { properties } = message; + if (!properties) { + properties = {}; + } + if (!properties.listData) { + properties.listData = { add: [], remove: [] }; + } + // message.properties.listData = message.properties.listData || { add: [], remove: [] }; + const fieldsId = fields?.id; + if (fieldsId === undefined) { + throw new InstrumentationError('No lead id passed in the payload.'); + } + if (action === 'insert') { + leadIdObj.insert.push({ id: fieldsId }); + properties.listData.add.push({ id: fieldsId }); + } else if (action === 'delete') { + leadIdObj.delete.push({ id: fieldsId }); + properties.listData.remove.push({ id: fieldsId }); + } else { + throw new InstrumentationError('Invalid action type'); + } + return input; + }); + const finalInput = [tokenisedInputs[0]]; + finalInput[0].metadata = finalMetadata; + finalInput[0].message.properties.listData.add = leadIdObj.insert; + finalInput[0].message.properties.listData.remove = leadIdObj.delete; + + return finalInput; +} + module.exports = { getIds, validateMessageType, + transformForRecordEvent, }; diff --git a/src/v0/destinations/marketo_static_list/util.test.js b/src/v0/destinations/marketo_static_list/util.test.js new file mode 100644 index 0000000000..b8bb1e216d --- /dev/null +++ b/src/v0/destinations/marketo_static_list/util.test.js @@ -0,0 +1,239 @@ +const { InstrumentationError } = require('../../util/errorTypes'); +const { + transformForRecordEvent, +} = require('../../../../src/v0/destinations/marketo_static_list/util'); + +describe('transformForRecordEvent', () => { + // Should correctly transform an input with action 'insert' + test('should correctly transform an input with action "insert"', () => { + const inputs = [ + { + message: { + fields: { id: 1 }, + action: 'insert', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata1', + }, + ]; + const leadIdObj = { insert: [], delete: [] }; + const expectedOutput = [ + { + message: { + fields: { id: 1 }, + action: 'insert', + properties: { listData: { add: [{ id: 1 }], remove: [] } }, + }, + metadata: ['metadata1'], + }, + ]; + + const result = transformForRecordEvent(inputs, leadIdObj); + + expect(result).toEqual(expectedOutput); + }); + + // Should correctly transform an input with action 'delete' + test('should correctly transform an input with action "delete"', () => { + const inputs = [ + { + message: { + fields: { id: 1 }, + action: 'delete', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata1', + }, + ]; + const leadIdObj = { insert: [], delete: [] }; + const expectedOutput = [ + { + message: { + fields: { id: 1 }, + action: 'delete', + properties: { listData: { add: [], remove: [{ id: 1 }] } }, + }, + metadata: ['metadata1'], + }, + ]; + + const result = transformForRecordEvent(inputs, leadIdObj); + + expect(result).toEqual(expectedOutput); + }); + + // Should raise an InstrumentationError if action is not valid + test('should raise an InstrumentationError if action is not valid', () => { + const inputs = [ + { + message: { + fields: { id: 1 }, + action: 'invalid', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata1', + }, + ]; + const leadIdObj = { insert: [], delete: [] }; + + expect(() => { + transformForRecordEvent(inputs, leadIdObj); + }).toThrow(InstrumentationError); + }); + + // Should raise an InstrumentationError on input with no fields + test('should raise an InstrumentationError on input with no fields', () => { + const inputs = [ + { + message: { + action: 'insert', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata1', + }, + ]; + const leadIdObj = { insert: [], delete: [] }; + + expect(() => { + transformForRecordEvent(inputs, leadIdObj); + }).toThrow(InstrumentationError); + }); + + // Should correctly transform an input with 2 insert payloads + test('should correctly transform an input with 2 insert payloads', () => { + const inputs = [ + { + message: { + type: 'record', + fields: { id: 1 }, + action: 'insert', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata1', + }, + { + message: { + type: 'record', + fields: { id: 2 }, + action: 'insert', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata2', + }, + ]; + const leadIdObj = { insert: [], delete: [] }; + const expectedOutput = [ + { + message: { + type: 'record', + fields: { id: 1 }, + action: 'insert', + properties: { listData: { add: [{ id: 1 }, { id: 2 }], remove: [] } }, + }, + metadata: ['metadata1', 'metadata2'], + }, + ]; + + const result = transformForRecordEvent(inputs, leadIdObj); + + expect(result).toEqual(expectedOutput); + }); + + // Should correctly transform an input with 2 delete payloads + test('should correctly transform an input with 2 delete payloads', () => { + const inputs = [ + { + message: { + type: 'record', + fields: { id: 1 }, + action: 'delete', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata1', + }, + { + message: { + type: 'record', + fields: { id: 2 }, + action: 'delete', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata2', + }, + ]; + const leadIdObj = { insert: [], delete: [] }; + const expectedOutput = [ + { + message: { + type: 'record', + fields: { id: 1 }, + action: 'delete', + properties: { listData: { remove: [{ id: 1 }, { id: 2 }], add: [] } }, + }, + metadata: ['metadata1', 'metadata2'], + }, + ]; + + const result = transformForRecordEvent(inputs, leadIdObj); + + expect(result).toEqual(expectedOutput); + }); + + // Should correctly transform an input with 2 insert and 2 delete payloads + test('should correctly transform an input with 2 insert and 2 delete payloads', () => { + const inputs = [ + { + message: { + type: 'record', + fields: { id: 1 }, + action: 'delete', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata1', + }, + { + message: { + type: 'record', + fields: { id: 2 }, + action: 'insert', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata2', + }, + { + message: { + type: 'record', + fields: { id: 3 }, + action: 'delete', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata3', + }, + { + message: { + type: 'record', + fields: { id: 4 }, + action: 'insert', + properties: { listData: { add: [], remove: [] } }, + }, + metadata: 'metadata4', + }, + ]; + const leadIdObj = { insert: [], delete: [] }; + const expectedOutput = [ + { + message: { + type: 'record', + fields: { id: 1 }, + action: 'delete', + properties: { listData: { remove: [{ id: 1 }, { id: 3 }], add: [{ id: 2 }, { id: 4 }] } }, + }, + metadata: ['metadata1', 'metadata2', 'metadata3', 'metadata4'], + }, + ]; + + const result = transformForRecordEvent(inputs, leadIdObj); + + expect(result).toEqual(expectedOutput); + }); +}); From 15a86ff63bf236d6a3a6cea4a33639b22501fadb Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Wed, 8 Nov 2023 18:16:27 +0530 Subject: [PATCH 06/15] chore: address commentsx2, handle mix of audiencelist and record events --- .../marketo_static_list/transform.js | 73 ++++--------------- .../destinations/marketo_static_list/util.js | 19 ++--- .../marketo_static_list/util.test.js | 4 + 3 files changed, 29 insertions(+), 67 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 1faae70443..0d6f266687 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -16,7 +16,7 @@ const { const { formatConfig, MAX_LEAD_IDS_SIZE } = require('./config'); const Cache = require('../../util/cache'); const { getAuthToken } = require('../marketo/transform'); -const { InstrumentationError, UnauthorizedError, AbortedError } = require('../../util/errorTypes'); +const { InstrumentationError, UnauthorizedError } = require('../../util/errorTypes'); const authCache = new Cache(AUTH_CACHE_TTL); // 1 hr @@ -94,41 +94,6 @@ const processEvent = (input) => { return response; }; -// function transformForRecordEvent(inputs, leadIdObj) { -// const finalMetadata = []; -// // iterate through each inputs metadata and create a final metadata -// const tokenisedInputs = inputs.map((input) => { -// const { message } = input; -// const { metadata } = input; -// finalMetadata.push(metadata); -// const { fields, action } = message; -// if (!message.properties) { -// message.properties = {}; -// } -// if (!message.properties.listData) { -// message.properties.listData = { add: [], remove: [] }; -// } -// // message.properties.listData = message.properties.listData || { add: [], remove: [] }; -// const fieldsId = fields.id; -// if (action === 'insert') { -// leadIdObj.insert.push({ id: fieldsId }); -// message.properties.listData.add.push({ id: fieldsId }); -// } else if (action === 'delete') { -// leadIdObj.delete.push({ id: fieldsId }); -// message.properties.listData.remove.push({ id: fieldsId }); -// } else { -// throw new InstrumentationError('Invalid action type'); -// } -// return input; -// }); -// const finalInput = [tokenisedInputs[0]]; -// finalInput[0].metadata = finalMetadata; -// finalInput[0].message.properties.listData.add = leadIdObj.insert; -// finalInput[0].message.properties.listData.remove = leadIdObj.delete; - -// return finalInput; -// } - const process = async (event) => { const token = await getAuthToken(formatConfig(event.destination)); @@ -143,24 +108,10 @@ const processRouterDest = async (inputs, reqMetadata) => { // Token needs to be generated for marketo which will be done on input level. // If destination information is not present Error should be thrown let token; - let respList; try { token = await getAuthToken(formatConfig(inputs[0].destination)); if (!token) { - throw new AbortedError('Could not retrieve authorisation token', 400); - // const errResp = { - // status: 400, - // message: 'Authorisation failed', - // responseTransformFailure: true, - // statTags: {}, - // }; - // const respEvents = getErrorRespEvents( - // inputs.map((input) => input.metadata), - // errResp.status, - // errResp.message, - // errResp.statTags, - // ); - // return [{ ...respEvents, destination: inputs?.[0]?.destination }]; + throw new InstrumentationError('Could not retrieve authorisation token', 400); } } catch (error) { // Not using handleRtTfSingleEventError here as this is for multiple events @@ -182,13 +133,19 @@ const processRouterDest = async (inputs, reqMetadata) => { insert: [], delete: [], }; - let finalInputForRecordEvent; - if (inputs[0].message.channel === 'sources' && inputs[0].message.type === 'record') { - finalInputForRecordEvent = transformForRecordEvent(tokenisedInputs, leadIdObj); - respList = await simpleProcessRouterDest(finalInputForRecordEvent, processEvent, reqMetadata); - } else { - respList = await simpleProcessRouterDest(tokenisedInputs, processEvent, reqMetadata); - } + // use lodash.groupby to group the inputs based on message type + const groupedInputs = lodash.groupBy(tokenisedInputs, (input) => input.message.type); + const finalInputForRecordEvent = transformForRecordEvent(groupedInputs.record, leadIdObj); + const transformedRecordEvent = await simpleProcessRouterDest(finalInputForRecordEvent, processEvent, reqMetadata); + const transformedAudienceEvent = await simpleProcessRouterDest(groupedInputs.audiencelist, processEvent, reqMetadata); + const respList = [...transformedRecordEvent, ...transformedAudienceEvent]; + + // if (inputs[0].message.channel === 'sources' && inputs[0].message.type === 'record') { + // finalInputForRecordEvent = transformForRecordEvent(tokenisedInputs, leadIdObj); + // respList = await simpleProcessRouterDest(finalInputForRecordEvent, processEvent, reqMetadata); + // } else { + // respList = await simpleProcessRouterDest(tokenisedInputs, processEvent, reqMetadata); + // } // [respList[0].metadata] = respList[0].metadata; return respList; }; diff --git a/src/v0/destinations/marketo_static_list/util.js b/src/v0/destinations/marketo_static_list/util.js index 4427891640..7fd20a56d6 100644 --- a/src/v0/destinations/marketo_static_list/util.js +++ b/src/v0/destinations/marketo_static_list/util.js @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/consistent-destructuring */ const { InstrumentationError } = require('../../util/errorTypes'); /** @@ -41,25 +42,25 @@ function transformForRecordEvent(inputs, leadIdObj) { const { message } = input; const { metadata } = input; finalMetadata.push(metadata); - const { fields, action } = message; - let { properties } = message; - if (!properties) { - properties = {}; + const { fields, action, type } = message; + if (type !== 'record') { + throw new InstrumentationError('Invalid message type, Supported message type is record.'); } - if (!properties.listData) { - properties.listData = { add: [], remove: [] }; + const { properties } = message; + if (!properties) { + message.properties = {}; } - // message.properties.listData = message.properties.listData || { add: [], remove: [] }; + message.properties.listData = message?.properties.listData || { add: [], remove: [] }; const fieldsId = fields?.id; if (fieldsId === undefined) { throw new InstrumentationError('No lead id passed in the payload.'); } if (action === 'insert') { leadIdObj.insert.push({ id: fieldsId }); - properties.listData.add.push({ id: fieldsId }); + message.properties.listData.add.push({ id: fieldsId }); } else if (action === 'delete') { leadIdObj.delete.push({ id: fieldsId }); - properties.listData.remove.push({ id: fieldsId }); + message.properties.listData.remove.push({ id: fieldsId }); } else { throw new InstrumentationError('Invalid action type'); } diff --git a/src/v0/destinations/marketo_static_list/util.test.js b/src/v0/destinations/marketo_static_list/util.test.js index b8bb1e216d..c9b1f803a5 100644 --- a/src/v0/destinations/marketo_static_list/util.test.js +++ b/src/v0/destinations/marketo_static_list/util.test.js @@ -9,6 +9,7 @@ describe('transformForRecordEvent', () => { const inputs = [ { message: { + type: 'record', fields: { id: 1 }, action: 'insert', properties: { listData: { add: [], remove: [] } }, @@ -20,6 +21,7 @@ describe('transformForRecordEvent', () => { const expectedOutput = [ { message: { + type: 'record', fields: { id: 1 }, action: 'insert', properties: { listData: { add: [{ id: 1 }], remove: [] } }, @@ -38,6 +40,7 @@ describe('transformForRecordEvent', () => { const inputs = [ { message: { + type: 'record', fields: { id: 1 }, action: 'delete', properties: { listData: { add: [], remove: [] } }, @@ -49,6 +52,7 @@ describe('transformForRecordEvent', () => { const expectedOutput = [ { message: { + type: 'record', fields: { id: 1 }, action: 'delete', properties: { listData: { add: [], remove: [{ id: 1 }] } }, From bb41537b070db1990d1879a7881c227459e0336e Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:58:19 +0530 Subject: [PATCH 07/15] chore: address commentsx2, handle mix of audiencelist and record events --- .../marketo_static_list/transform.js | 31 ++++--- .../marketo_static_list/router/data.ts | 81 ------------------- 2 files changed, 19 insertions(+), 93 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 0d6f266687..d70d2b5a2e 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -111,7 +111,7 @@ const processRouterDest = async (inputs, reqMetadata) => { try { token = await getAuthToken(formatConfig(inputs[0].destination)); if (!token) { - throw new InstrumentationError('Could not retrieve authorisation token', 400); + throw new UnauthorizedError('Could not retrieve authorisation token'); } } catch (error) { // Not using handleRtTfSingleEventError here as this is for multiple events @@ -134,19 +134,26 @@ const processRouterDest = async (inputs, reqMetadata) => { delete: [], }; // use lodash.groupby to group the inputs based on message type + let transformedRecordEvent = []; + let transformedAudienceEvent = []; const groupedInputs = lodash.groupBy(tokenisedInputs, (input) => input.message.type); - const finalInputForRecordEvent = transformForRecordEvent(groupedInputs.record, leadIdObj); - const transformedRecordEvent = await simpleProcessRouterDest(finalInputForRecordEvent, processEvent, reqMetadata); - const transformedAudienceEvent = await simpleProcessRouterDest(groupedInputs.audiencelist, processEvent, reqMetadata); - const respList = [...transformedRecordEvent, ...transformedAudienceEvent]; - // if (inputs[0].message.channel === 'sources' && inputs[0].message.type === 'record') { - // finalInputForRecordEvent = transformForRecordEvent(tokenisedInputs, leadIdObj); - // respList = await simpleProcessRouterDest(finalInputForRecordEvent, processEvent, reqMetadata); - // } else { - // respList = await simpleProcessRouterDest(tokenisedInputs, processEvent, reqMetadata); - // } - // [respList[0].metadata] = respList[0].metadata; + if (groupedInputs.record && groupedInputs.record.length > 0) { + const finalInputForRecordEvent = transformForRecordEvent(groupedInputs.record, leadIdObj); + transformedRecordEvent = await simpleProcessRouterDest( + finalInputForRecordEvent, + processEvent, + reqMetadata, + ); + } + if (groupedInputs.audiencelist && groupedInputs.audiencelist.length > 0) { + transformedAudienceEvent = await simpleProcessRouterDest( + groupedInputs.audiencelist, + processEvent, + reqMetadata, + ); + } + const respList = [...transformedRecordEvent, ...transformedAudienceEvent]; return respList; }; diff --git a/test/integrations/destinations/marketo_static_list/router/data.ts b/test/integrations/destinations/marketo_static_list/router/data.ts index e0d853e1f2..f086b41641 100644 --- a/test/integrations/destinations/marketo_static_list/router/data.ts +++ b/test/integrations/destinations/marketo_static_list/router/data.ts @@ -1172,50 +1172,6 @@ export const data = [ jobId: 2, }, }, - { - destination: { - ID: '1zia9wKshXt80YksLmUdJnr7IHI', - Name: 'test_marketo', - DestinationDefinition: { - ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', - Name: 'MARKETO', - DisplayName: 'Marketo', - transformAt: 'processor', - transformAtV1: 'processor', - }, - Config: { - clientId: 'marketo_client_id_success', - clientSecret: 'marketo_client_secret_success', - accountId: 'marketo_acct_id_success', - staticListId: 1234, - }, - Enabled: true, - Transformations: [], - IsProcessorEnabled: true, - }, - message: { - userId: 'user 1', - anonymousId: 'anon-id-new', - event: 'event1', - type: 'track', - properties: { - listData: { - remove: [1], - }, - enablePartialFailure: true, - }, - context: { - ip: '14.5.67.21', - library: { - name: 'http', - }, - }, - timestamp: '2020-02-02T00:23:09.544Z', - }, - metadata: { - jobId: 3, - }, - }, ], destType: 'marketo_static_list', }, @@ -1371,43 +1327,6 @@ export const data = [ IsProcessorEnabled: true, }, }, - { - metadata: [ - { - destInfo: { - authKey: '1zia9wKshXt80YksLmUdJnr7IHI', - }, - jobId: 3, - }, - ], - destination: { - ID: '1zia9wKshXt80YksLmUdJnr7IHI', - Name: 'test_marketo', - DestinationDefinition: { - ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', - Name: 'MARKETO', - DisplayName: 'Marketo', - transformAt: 'processor', - transformAtV1: 'processor', - }, - Config: { - clientId: 'marketo_client_id_success', - clientSecret: 'marketo_client_secret_success', - accountId: 'marketo_acct_id_success', - staticListId: 1234, - }, - Enabled: true, - Transformations: [], - IsProcessorEnabled: true, - }, - batched: false, - statusCode: 400, - error: 'Event type track is not supported', - statTags: { - errorCategory: 'dataValidation', - errorType: 'instrumentation', - }, - }, ], }, }, From 761a1f840fa4ddcdd781f72db974a97435d57bb1 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:10:46 +0530 Subject: [PATCH 08/15] chore: add router test --- .../marketo_static_list/router/data.ts | 1735 +++++++++++++++++ 1 file changed, 1735 insertions(+) diff --git a/test/integrations/destinations/marketo_static_list/router/data.ts b/test/integrations/destinations/marketo_static_list/router/data.ts index f086b41641..abf37c3294 100644 --- a/test/integrations/destinations/marketo_static_list/router/data.ts +++ b/test/integrations/destinations/marketo_static_list/router/data.ts @@ -1332,4 +1332,1739 @@ export const data = [ }, }, }, + { + name: 'marketo_static_list', + description: 'Test 1', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1001, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'MARKETO_STATIC_LIST-leadId', + identifierType: 'id', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 1, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1002, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'MARKETO_STATIC_LIST-leadId', + identifierType: 'id', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 2, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1003, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'MARKETO_STATIC_LIST-leadId', + identifierType: 'id', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 3, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + identifierType: 'id', + type: 'MARKETO_STATIC_LIST-leadId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2001', + }, + type: 'record', + }, + metadata: { + jobId: 4, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + identifierType: 'id', + type: 'MARKETO_STATIC_LIST-leadId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2002', + }, + type: 'record', + }, + metadata: { + jobId: 5, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + identifierType: 'id', + type: 'MARKETO_STATIC_LIST-leadId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2003', + }, + type: 'record', + }, + metadata: { + jobId: 6, + }, + }, + { + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo_al', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 101, + }, + { + id: 102, + }, + { + id: 103, + }, + ], + remove: [ + { + id: 4, + }, + { + id: 5, + }, + { + id: 6, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 7, + }, + }, + { + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo_al', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 0, + }, + { + id: 1, + }, + { + id: 2, + }, + { + id: 3, + }, + { + id: 4, + }, + { + id: 5, + }, + { + id: 6, + }, + { + id: 7, + }, + { + id: 8, + }, + { + id: 9, + }, + { + id: 10, + }, + { + id: 11, + }, + { + id: 12, + }, + { + id: 13, + }, + { + id: 14, + }, + { + id: 15, + }, + { + id: 16, + }, + { + id: 17, + }, + { + id: 18, + }, + { + id: 19, + }, + { + id: 20, + }, + { + id: 21, + }, + { + id: 22, + }, + { + id: 23, + }, + { + id: 24, + }, + { + id: 25, + }, + { + id: 26, + }, + { + id: 27, + }, + { + id: 28, + }, + { + id: 29, + }, + { + id: 30, + }, + { + id: 31, + }, + { + id: 32, + }, + { + id: 33, + }, + { + id: 34, + }, + { + id: 35, + }, + { + id: 36, + }, + { + id: 37, + }, + { + id: 38, + }, + { + id: 39, + }, + { + id: 40, + }, + { + id: 41, + }, + { + id: 42, + }, + { + id: 43, + }, + { + id: 44, + }, + { + id: 45, + }, + { + id: 46, + }, + { + id: 47, + }, + { + id: 48, + }, + { + id: 49, + }, + { + id: 50, + }, + { + id: 51, + }, + { + id: 52, + }, + { + id: 53, + }, + { + id: 54, + }, + { + id: 55, + }, + { + id: 56, + }, + { + id: 57, + }, + { + id: 58, + }, + { + id: 59, + }, + { + id: 60, + }, + { + id: 61, + }, + { + id: 62, + }, + { + id: 63, + }, + { + id: 64, + }, + { + id: 65, + }, + { + id: 66, + }, + { + id: 67, + }, + { + id: 68, + }, + { + id: 69, + }, + { + id: 70, + }, + { + id: 71, + }, + { + id: 72, + }, + { + id: 73, + }, + { + id: 74, + }, + { + id: 75, + }, + { + id: 76, + }, + { + id: 77, + }, + { + id: 78, + }, + { + id: 79, + }, + { + id: 80, + }, + { + id: 81, + }, + { + id: 82, + }, + { + id: 83, + }, + { + id: 84, + }, + { + id: 85, + }, + { + id: 86, + }, + { + id: 87, + }, + { + id: 88, + }, + { + id: 89, + }, + { + id: 90, + }, + { + id: 91, + }, + { + id: 92, + }, + { + id: 93, + }, + { + id: 94, + }, + { + id: 95, + }, + { + id: 96, + }, + { + id: 97, + }, + { + id: 98, + }, + { + id: 99, + }, + { + id: 100, + }, + { + id: 101, + }, + { + id: 102, + }, + { + id: 103, + }, + { + id: 104, + }, + { + id: 105, + }, + { + id: 106, + }, + { + id: 107, + }, + { + id: 108, + }, + { + id: 109, + }, + { + id: 110, + }, + { + id: 111, + }, + { + id: 112, + }, + { + id: 113, + }, + { + id: 114, + }, + { + id: 115, + }, + { + id: 116, + }, + { + id: 117, + }, + { + id: 118, + }, + { + id: 119, + }, + { + id: 120, + }, + { + id: 121, + }, + { + id: 122, + }, + { + id: 123, + }, + { + id: 124, + }, + { + id: 125, + }, + { + id: 126, + }, + { + id: 127, + }, + { + id: 128, + }, + { + id: 129, + }, + { + id: 130, + }, + { + id: 131, + }, + { + id: 132, + }, + { + id: 133, + }, + { + id: 134, + }, + { + id: 135, + }, + { + id: 136, + }, + { + id: 137, + }, + { + id: 138, + }, + { + id: 139, + }, + { + id: 140, + }, + { + id: 141, + }, + { + id: 142, + }, + { + id: 143, + }, + { + id: 144, + }, + { + id: 145, + }, + { + id: 146, + }, + { + id: 147, + }, + { + id: 148, + }, + { + id: 149, + }, + { + id: 150, + }, + { + id: 151, + }, + { + id: 152, + }, + { + id: 153, + }, + { + id: 154, + }, + { + id: 155, + }, + { + id: 156, + }, + { + id: 157, + }, + { + id: 158, + }, + { + id: 159, + }, + { + id: 160, + }, + { + id: 161, + }, + { + id: 162, + }, + { + id: 163, + }, + { + id: 164, + }, + { + id: 165, + }, + { + id: 166, + }, + { + id: 167, + }, + { + id: 168, + }, + { + id: 169, + }, + { + id: 170, + }, + { + id: 171, + }, + { + id: 172, + }, + { + id: 173, + }, + { + id: 174, + }, + { + id: 175, + }, + { + id: 176, + }, + { + id: 177, + }, + { + id: 178, + }, + { + id: 179, + }, + { + id: 180, + }, + { + id: 181, + }, + { + id: 182, + }, + { + id: 183, + }, + { + id: 184, + }, + { + id: 185, + }, + { + id: 186, + }, + { + id: 187, + }, + { + id: 188, + }, + { + id: 189, + }, + { + id: 190, + }, + { + id: 191, + }, + { + id: 192, + }, + { + id: 193, + }, + { + id: 194, + }, + { + id: 195, + }, + { + id: 196, + }, + { + id: 197, + }, + { + id: 198, + }, + { + id: 199, + }, + { + id: 200, + }, + { + id: 201, + }, + { + id: 202, + }, + { + id: 203, + }, + { + id: 204, + }, + { + id: 205, + }, + { + id: 206, + }, + { + id: 207, + }, + { + id: 208, + }, + { + id: 209, + }, + { + id: 210, + }, + { + id: 211, + }, + { + id: 212, + }, + { + id: 213, + }, + { + id: 214, + }, + { + id: 215, + }, + { + id: 216, + }, + { + id: 217, + }, + { + id: 218, + }, + { + id: 219, + }, + { + id: 220, + }, + { + id: 221, + }, + { + id: 222, + }, + { + id: 223, + }, + { + id: 224, + }, + { + id: 225, + }, + { + id: 226, + }, + { + id: 227, + }, + { + id: 228, + }, + { + id: 229, + }, + { + id: 230, + }, + { + id: 231, + }, + { + id: 232, + }, + { + id: 233, + }, + { + id: 234, + }, + { + id: 235, + }, + { + id: 236, + }, + { + id: 237, + }, + { + id: 238, + }, + { + id: 239, + }, + { + id: 240, + }, + { + id: 241, + }, + { + id: 242, + }, + { + id: 243, + }, + { + id: 244, + }, + { + id: 245, + }, + { + id: 246, + }, + { + id: 247, + }, + { + id: 248, + }, + { + id: 249, + }, + { + id: 250, + }, + { + id: 251, + }, + { + id: 252, + }, + { + id: 253, + }, + { + id: 254, + }, + { + id: 255, + }, + { + id: 256, + }, + { + id: 257, + }, + { + id: 258, + }, + { + id: 259, + }, + { + id: 260, + }, + { + id: 261, + }, + { + id: 262, + }, + { + id: 263, + }, + { + id: 264, + }, + { + id: 265, + }, + { + id: 266, + }, + { + id: 267, + }, + { + id: 268, + }, + { + id: 269, + }, + { + id: 270, + }, + { + id: 271, + }, + { + id: 272, + }, + { + id: 273, + }, + { + id: 274, + }, + { + id: 275, + }, + { + id: 276, + }, + { + id: 277, + }, + { + id: 278, + }, + { + id: 279, + }, + { + id: 280, + }, + { + id: 281, + }, + { + id: 282, + }, + { + id: 283, + }, + { + id: 284, + }, + { + id: 285, + }, + { + id: 286, + }, + { + id: 287, + }, + { + id: 288, + }, + { + id: 289, + }, + { + id: 290, + }, + { + id: 291, + }, + { + id: 292, + }, + { + id: 293, + }, + { + id: 294, + }, + { + id: 295, + }, + { + id: 296, + }, + { + id: 297, + }, + { + id: 298, + }, + { + id: 299, + }, + { + id: 300, + }, + { + id: 301, + }, + { + id: 302, + }, + { + id: 303, + }, + { + id: 304, + }, + { + id: 305, + }, + { + id: 306, + }, + { + id: 307, + }, + { + id: 308, + }, + { + id: 309, + }, + { + id: 310, + }, + { + id: 311, + }, + { + id: 312, + }, + { + id: 313, + }, + { + id: 314, + }, + { + id: 315, + }, + { + id: 316, + }, + { + id: 317, + }, + { + id: 318, + }, + { + id: 319, + }, + { + id: 320, + }, + { + id: 321, + }, + { + id: 322, + }, + { + id: 323, + }, + { + id: 324, + }, + { + id: 325, + }, + { + id: 326, + }, + { + id: 327, + }, + { + id: 328, + }, + { + id: 329, + }, + { + id: 330, + }, + { + id: 331, + }, + { + id: 332, + }, + { + id: 333, + }, + { + id: 334, + }, + { + id: 335, + }, + { + id: 336, + }, + { + id: 337, + }, + { + id: 338, + }, + { + id: 339, + }, + { + id: 340, + }, + { + id: 341, + }, + { + id: 342, + }, + { + id: 343, + }, + { + id: 344, + }, + { + id: 345, + }, + { + id: 346, + }, + { + id: 347, + }, + { + id: 348, + }, + { + id: 349, + }, + { + id: 350, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 8, + }, + }, + ], + destType: 'marketo_static_list', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=2001&id=2002&id=2003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=1001&id=1002&id=1003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 2, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 3, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 4, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 5, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 6, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=101&id=102&id=103', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 7, + destInfo: { + authKey: '1zia9wKshXt80YksLmUdJnr7IHI', + }, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo_al', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 8, + destInfo: { + authKey: '1zia9wKshXt80YksLmUdJnr7IHI', + }, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo_al', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + ], + }, + }, + }, + }, ]; From 996dcc8e27899f4f91979ff289358fec8dd7a032 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Wed, 22 Nov 2023 19:25:02 +0530 Subject: [PATCH 09/15] fix: record to audience handling (#2813) --- .../marketo_static_list/testData/constants.js | 51 + .../marketo_static_list/testData/testData.js | 3092 +++++++++++++++++ .../marketo_static_list/transform.js | 90 +- .../marketo_static_list/transform.test.js | 54 + .../marketo_static_list/transformV2.js | 135 + .../destinations/marketo_static_list/util.js | 40 - .../marketo_static_list/util.test.js | 243 -- .../marketo_static_list/router/data.ts | 632 +++- 8 files changed, 3983 insertions(+), 354 deletions(-) create mode 100644 src/v0/destinations/marketo_static_list/testData/constants.js create mode 100644 src/v0/destinations/marketo_static_list/testData/testData.js create mode 100644 src/v0/destinations/marketo_static_list/transform.test.js create mode 100644 src/v0/destinations/marketo_static_list/transformV2.js delete mode 100644 src/v0/destinations/marketo_static_list/util.test.js diff --git a/src/v0/destinations/marketo_static_list/testData/constants.js b/src/v0/destinations/marketo_static_list/testData/constants.js new file mode 100644 index 0000000000..0ba27c277d --- /dev/null +++ b/src/v0/destinations/marketo_static_list/testData/constants.js @@ -0,0 +1,51 @@ +const EXTERNAL_ID = 'marketoStaticListId'; +const TOKEN = 'Bearer access_token_success'; +const CONTENT_TYPE = 'application/json'; +const DEST_OBJECT = { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, +}; +const DEST_DEFINITION = { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', +}; +const MESSAGE_SOURCES_CONTEXT = { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', +}; +const DEST_CONFIG = { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, +}; + +module.exports = { + EXTERNAL_ID, + TOKEN, + CONTENT_TYPE, + DEST_OBJECT, + DEST_DEFINITION, + MESSAGE_SOURCES_CONTEXT, + DEST_CONFIG, +}; diff --git a/src/v0/destinations/marketo_static_list/testData/testData.js b/src/v0/destinations/marketo_static_list/testData/testData.js new file mode 100644 index 0000000000..d0d5f24cb5 --- /dev/null +++ b/src/v0/destinations/marketo_static_list/testData/testData.js @@ -0,0 +1,3092 @@ +const { + EXTERNAL_ID, + TOKEN, + CONTENT_TYPE, + DEST_OBJECT, + DEST_DEFINITION, + MESSAGE_SOURCES_CONTEXT, + DEST_CONFIG, +} = require('./constants'); + +const recordInputs = [ + { + destination: DEST_OBJECT, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1001, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 1, + }, + }, + { + destination: DEST_OBJECT, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1002, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: 'id002', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 2, + }, + }, + { + destination: DEST_OBJECT, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1003, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 3, + }, + }, + { + destination: DEST_OBJECT, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id002', + type: EXTERNAL_ID, + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2001', + }, + type: 'record', + }, + metadata: { + jobId: 4, + }, + }, +]; + +const recordOutput = [ + { + batchedRequest: [], + metadata: [], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=1001&id=1003', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + }, + { + jobId: 3, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=2001', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 4, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=1002', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 2, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, +]; + +const audiencelistInputs = [ + { + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 501, + }, + { + id: 502, + }, + { + id: 503, + }, + ], + remove: [ + { + id: 704, + }, + { + id: 705, + }, + { + id: 706, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 1, + }, + }, + { + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 0, + }, + { + id: 1, + }, + { + id: 2, + }, + { + id: 3, + }, + { + id: 4, + }, + { + id: 5, + }, + { + id: 6, + }, + { + id: 7, + }, + { + id: 8, + }, + { + id: 9, + }, + { + id: 10, + }, + { + id: 11, + }, + { + id: 12, + }, + { + id: 13, + }, + { + id: 14, + }, + { + id: 15, + }, + { + id: 16, + }, + { + id: 17, + }, + { + id: 18, + }, + { + id: 19, + }, + { + id: 20, + }, + { + id: 21, + }, + { + id: 22, + }, + { + id: 23, + }, + { + id: 24, + }, + { + id: 25, + }, + { + id: 26, + }, + { + id: 27, + }, + { + id: 28, + }, + { + id: 29, + }, + { + id: 30, + }, + { + id: 31, + }, + { + id: 32, + }, + { + id: 33, + }, + { + id: 34, + }, + { + id: 35, + }, + { + id: 36, + }, + { + id: 37, + }, + { + id: 38, + }, + { + id: 39, + }, + { + id: 40, + }, + { + id: 41, + }, + { + id: 42, + }, + { + id: 43, + }, + { + id: 44, + }, + { + id: 45, + }, + { + id: 46, + }, + { + id: 47, + }, + { + id: 48, + }, + { + id: 49, + }, + { + id: 50, + }, + { + id: 51, + }, + { + id: 52, + }, + { + id: 53, + }, + { + id: 54, + }, + { + id: 55, + }, + { + id: 56, + }, + { + id: 57, + }, + { + id: 58, + }, + { + id: 59, + }, + { + id: 60, + }, + { + id: 61, + }, + { + id: 62, + }, + { + id: 63, + }, + { + id: 64, + }, + { + id: 65, + }, + { + id: 66, + }, + { + id: 67, + }, + { + id: 68, + }, + { + id: 69, + }, + { + id: 70, + }, + { + id: 71, + }, + { + id: 72, + }, + { + id: 73, + }, + { + id: 74, + }, + { + id: 75, + }, + { + id: 76, + }, + { + id: 77, + }, + { + id: 78, + }, + { + id: 79, + }, + { + id: 80, + }, + { + id: 81, + }, + { + id: 82, + }, + { + id: 83, + }, + { + id: 84, + }, + { + id: 85, + }, + { + id: 86, + }, + { + id: 87, + }, + { + id: 88, + }, + { + id: 89, + }, + { + id: 90, + }, + { + id: 91, + }, + { + id: 92, + }, + { + id: 93, + }, + { + id: 94, + }, + { + id: 95, + }, + { + id: 96, + }, + { + id: 97, + }, + { + id: 98, + }, + { + id: 99, + }, + { + id: 100, + }, + { + id: 101, + }, + { + id: 102, + }, + { + id: 103, + }, + { + id: 104, + }, + { + id: 105, + }, + { + id: 106, + }, + { + id: 107, + }, + { + id: 108, + }, + { + id: 109, + }, + { + id: 110, + }, + { + id: 111, + }, + { + id: 112, + }, + { + id: 113, + }, + { + id: 114, + }, + { + id: 115, + }, + { + id: 116, + }, + { + id: 117, + }, + { + id: 118, + }, + { + id: 119, + }, + { + id: 120, + }, + { + id: 121, + }, + { + id: 122, + }, + { + id: 123, + }, + { + id: 124, + }, + { + id: 125, + }, + { + id: 126, + }, + { + id: 127, + }, + { + id: 128, + }, + { + id: 129, + }, + { + id: 130, + }, + { + id: 131, + }, + { + id: 132, + }, + { + id: 133, + }, + { + id: 134, + }, + { + id: 135, + }, + { + id: 136, + }, + { + id: 137, + }, + { + id: 138, + }, + { + id: 139, + }, + { + id: 140, + }, + { + id: 141, + }, + { + id: 142, + }, + { + id: 143, + }, + { + id: 144, + }, + { + id: 145, + }, + { + id: 146, + }, + { + id: 147, + }, + { + id: 148, + }, + { + id: 149, + }, + { + id: 150, + }, + { + id: 151, + }, + { + id: 152, + }, + { + id: 153, + }, + { + id: 154, + }, + { + id: 155, + }, + { + id: 156, + }, + { + id: 157, + }, + { + id: 158, + }, + { + id: 159, + }, + { + id: 160, + }, + { + id: 161, + }, + { + id: 162, + }, + { + id: 163, + }, + { + id: 164, + }, + { + id: 165, + }, + { + id: 166, + }, + { + id: 167, + }, + { + id: 168, + }, + { + id: 169, + }, + { + id: 170, + }, + { + id: 171, + }, + { + id: 172, + }, + { + id: 173, + }, + { + id: 174, + }, + { + id: 175, + }, + { + id: 176, + }, + { + id: 177, + }, + { + id: 178, + }, + { + id: 179, + }, + { + id: 180, + }, + { + id: 181, + }, + { + id: 182, + }, + { + id: 183, + }, + { + id: 184, + }, + { + id: 185, + }, + { + id: 186, + }, + { + id: 187, + }, + { + id: 188, + }, + { + id: 189, + }, + { + id: 190, + }, + { + id: 191, + }, + { + id: 192, + }, + { + id: 193, + }, + { + id: 194, + }, + { + id: 195, + }, + { + id: 196, + }, + { + id: 197, + }, + { + id: 198, + }, + { + id: 199, + }, + { + id: 200, + }, + { + id: 201, + }, + { + id: 202, + }, + { + id: 203, + }, + { + id: 204, + }, + { + id: 205, + }, + { + id: 206, + }, + { + id: 207, + }, + { + id: 208, + }, + { + id: 209, + }, + { + id: 210, + }, + { + id: 211, + }, + { + id: 212, + }, + { + id: 213, + }, + { + id: 214, + }, + { + id: 215, + }, + { + id: 216, + }, + { + id: 217, + }, + { + id: 218, + }, + { + id: 219, + }, + { + id: 220, + }, + { + id: 221, + }, + { + id: 222, + }, + { + id: 223, + }, + { + id: 224, + }, + { + id: 225, + }, + { + id: 226, + }, + { + id: 227, + }, + { + id: 228, + }, + { + id: 229, + }, + { + id: 230, + }, + { + id: 231, + }, + { + id: 232, + }, + { + id: 233, + }, + { + id: 234, + }, + { + id: 235, + }, + { + id: 236, + }, + { + id: 237, + }, + { + id: 238, + }, + { + id: 239, + }, + { + id: 240, + }, + { + id: 241, + }, + { + id: 242, + }, + { + id: 243, + }, + { + id: 244, + }, + { + id: 245, + }, + { + id: 246, + }, + { + id: 247, + }, + { + id: 248, + }, + { + id: 249, + }, + { + id: 250, + }, + { + id: 251, + }, + { + id: 252, + }, + { + id: 253, + }, + { + id: 254, + }, + { + id: 255, + }, + { + id: 256, + }, + { + id: 257, + }, + { + id: 258, + }, + { + id: 259, + }, + { + id: 260, + }, + { + id: 261, + }, + { + id: 262, + }, + { + id: 263, + }, + { + id: 264, + }, + { + id: 265, + }, + { + id: 266, + }, + { + id: 267, + }, + { + id: 268, + }, + { + id: 269, + }, + { + id: 270, + }, + { + id: 271, + }, + { + id: 272, + }, + { + id: 273, + }, + { + id: 274, + }, + { + id: 275, + }, + { + id: 276, + }, + { + id: 277, + }, + { + id: 278, + }, + { + id: 279, + }, + { + id: 280, + }, + { + id: 281, + }, + { + id: 282, + }, + { + id: 283, + }, + { + id: 284, + }, + { + id: 285, + }, + { + id: 286, + }, + { + id: 287, + }, + { + id: 288, + }, + { + id: 289, + }, + { + id: 290, + }, + { + id: 291, + }, + { + id: 292, + }, + { + id: 293, + }, + { + id: 294, + }, + { + id: 295, + }, + { + id: 296, + }, + { + id: 297, + }, + { + id: 298, + }, + { + id: 299, + }, + { + id: 300, + }, + { + id: 301, + }, + { + id: 302, + }, + { + id: 303, + }, + { + id: 304, + }, + { + id: 305, + }, + { + id: 306, + }, + { + id: 307, + }, + { + id: 308, + }, + { + id: 309, + }, + { + id: 310, + }, + { + id: 311, + }, + { + id: 312, + }, + { + id: 313, + }, + { + id: 314, + }, + { + id: 315, + }, + { + id: 316, + }, + { + id: 317, + }, + { + id: 318, + }, + { + id: 319, + }, + { + id: 320, + }, + { + id: 321, + }, + { + id: 322, + }, + { + id: 323, + }, + { + id: 324, + }, + { + id: 325, + }, + { + id: 326, + }, + { + id: 327, + }, + { + id: 328, + }, + { + id: 329, + }, + { + id: 330, + }, + { + id: 331, + }, + { + id: 332, + }, + { + id: 333, + }, + { + id: 334, + }, + { + id: 335, + }, + { + id: 336, + }, + { + id: 337, + }, + { + id: 338, + }, + { + id: 339, + }, + { + id: 340, + }, + { + id: 341, + }, + { + id: 342, + }, + { + id: 343, + }, + { + id: 344, + }, + { + id: 345, + }, + { + id: 346, + }, + { + id: 347, + }, + { + id: 348, + }, + { + id: 349, + }, + { + id: 350, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 2, + }, + }, +]; + +const reqMetadata = { + namespace: 'Unknown', + cluster: 'Unknown', + features: {}, +}; + +const largeRecordOutput = [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=351&id=352&id=353&id=354&id=355&id=356&id=357&id=358&id=359&id=360&id=361&id=362&id=363&id=364&id=365&id=366&id=367&id=368&id=369&id=370&id=371&id=372&id=373&id=374&id=375&id=376&id=377&id=378&id=379&id=380&id=381&id=382&id=383&id=384&id=385&id=386&id=387&id=388&id=389&id=390&id=391&id=392&id=393&id=394&id=395&id=396&id=397&id=398&id=399&id=400&id=401&id=402&id=403&id=404&id=405&id=406&id=407&id=408&id=409&id=410&id=411&id=412&id=413&id=414&id=415&id=416&id=417&id=418&id=419&id=420&id=421&id=422&id=423&id=424&id=425&id=426&id=427&id=428&id=429&id=430&id=431&id=432&id=433&id=434&id=435&id=436&id=437&id=438&id=439&id=440&id=441&id=442&id=443&id=444&id=445&id=446&id=447&id=448&id=449&id=450', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 351, + }, + { + jobId: 352, + }, + { + jobId: 353, + }, + { + jobId: 354, + }, + { + jobId: 355, + }, + { + jobId: 356, + }, + { + jobId: 357, + }, + { + jobId: 358, + }, + { + jobId: 359, + }, + { + jobId: 360, + }, + { + jobId: 361, + }, + { + jobId: 362, + }, + { + jobId: 363, + }, + { + jobId: 364, + }, + { + jobId: 365, + }, + { + jobId: 366, + }, + { + jobId: 367, + }, + { + jobId: 368, + }, + { + jobId: 369, + }, + { + jobId: 370, + }, + { + jobId: 371, + }, + { + jobId: 372, + }, + { + jobId: 373, + }, + { + jobId: 374, + }, + { + jobId: 375, + }, + { + jobId: 376, + }, + { + jobId: 377, + }, + { + jobId: 378, + }, + { + jobId: 379, + }, + { + jobId: 380, + }, + { + jobId: 381, + }, + { + jobId: 382, + }, + { + jobId: 383, + }, + { + jobId: 384, + }, + { + jobId: 385, + }, + { + jobId: 386, + }, + { + jobId: 387, + }, + { + jobId: 388, + }, + { + jobId: 389, + }, + { + jobId: 390, + }, + { + jobId: 391, + }, + { + jobId: 392, + }, + { + jobId: 393, + }, + { + jobId: 394, + }, + { + jobId: 395, + }, + { + jobId: 396, + }, + { + jobId: 397, + }, + { + jobId: 398, + }, + { + jobId: 399, + }, + { + jobId: 400, + }, + { + jobId: 401, + }, + { + jobId: 402, + }, + { + jobId: 403, + }, + { + jobId: 404, + }, + { + jobId: 405, + }, + { + jobId: 406, + }, + { + jobId: 407, + }, + { + jobId: 408, + }, + { + jobId: 409, + }, + { + jobId: 410, + }, + { + jobId: 411, + }, + { + jobId: 412, + }, + { + jobId: 413, + }, + { + jobId: 414, + }, + { + jobId: 415, + }, + { + jobId: 416, + }, + { + jobId: 417, + }, + { + jobId: 418, + }, + { + jobId: 419, + }, + { + jobId: 420, + }, + { + jobId: 421, + }, + { + jobId: 422, + }, + { + jobId: 423, + }, + { + jobId: 424, + }, + { + jobId: 425, + }, + { + jobId: 426, + }, + { + jobId: 427, + }, + { + jobId: 428, + }, + { + jobId: 429, + }, + { + jobId: 430, + }, + { + jobId: 431, + }, + { + jobId: 432, + }, + { + jobId: 433, + }, + { + jobId: 434, + }, + { + jobId: 435, + }, + { + jobId: 436, + }, + { + jobId: 437, + }, + { + jobId: 438, + }, + { + jobId: 439, + }, + { + jobId: 440, + }, + { + jobId: 441, + }, + { + jobId: 442, + }, + { + jobId: 443, + }, + { + jobId: 444, + }, + { + jobId: 445, + }, + { + jobId: 446, + }, + { + jobId: 447, + }, + { + jobId: 448, + }, + { + jobId: 449, + }, + { + jobId: 450, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299&id=300', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + }, + { + jobId: 2, + }, + { + jobId: 3, + }, + { + jobId: 4, + }, + { + jobId: 5, + }, + { + jobId: 6, + }, + { + jobId: 7, + }, + { + jobId: 8, + }, + { + jobId: 9, + }, + { + jobId: 10, + }, + { + jobId: 11, + }, + { + jobId: 12, + }, + { + jobId: 13, + }, + { + jobId: 14, + }, + { + jobId: 15, + }, + { + jobId: 16, + }, + { + jobId: 17, + }, + { + jobId: 18, + }, + { + jobId: 19, + }, + { + jobId: 20, + }, + { + jobId: 21, + }, + { + jobId: 22, + }, + { + jobId: 23, + }, + { + jobId: 24, + }, + { + jobId: 25, + }, + { + jobId: 26, + }, + { + jobId: 27, + }, + { + jobId: 28, + }, + { + jobId: 29, + }, + { + jobId: 30, + }, + { + jobId: 31, + }, + { + jobId: 32, + }, + { + jobId: 33, + }, + { + jobId: 34, + }, + { + jobId: 35, + }, + { + jobId: 36, + }, + { + jobId: 37, + }, + { + jobId: 38, + }, + { + jobId: 39, + }, + { + jobId: 40, + }, + { + jobId: 41, + }, + { + jobId: 42, + }, + { + jobId: 43, + }, + { + jobId: 44, + }, + { + jobId: 45, + }, + { + jobId: 46, + }, + { + jobId: 47, + }, + { + jobId: 48, + }, + { + jobId: 49, + }, + { + jobId: 50, + }, + { + jobId: 51, + }, + { + jobId: 52, + }, + { + jobId: 53, + }, + { + jobId: 54, + }, + { + jobId: 55, + }, + { + jobId: 56, + }, + { + jobId: 57, + }, + { + jobId: 58, + }, + { + jobId: 59, + }, + { + jobId: 60, + }, + { + jobId: 61, + }, + { + jobId: 62, + }, + { + jobId: 63, + }, + { + jobId: 64, + }, + { + jobId: 65, + }, + { + jobId: 66, + }, + { + jobId: 67, + }, + { + jobId: 68, + }, + { + jobId: 69, + }, + { + jobId: 70, + }, + { + jobId: 71, + }, + { + jobId: 72, + }, + { + jobId: 73, + }, + { + jobId: 74, + }, + { + jobId: 75, + }, + { + jobId: 76, + }, + { + jobId: 77, + }, + { + jobId: 78, + }, + { + jobId: 79, + }, + { + jobId: 80, + }, + { + jobId: 81, + }, + { + jobId: 82, + }, + { + jobId: 83, + }, + { + jobId: 84, + }, + { + jobId: 85, + }, + { + jobId: 86, + }, + { + jobId: 87, + }, + { + jobId: 88, + }, + { + jobId: 89, + }, + { + jobId: 90, + }, + { + jobId: 91, + }, + { + jobId: 92, + }, + { + jobId: 93, + }, + { + jobId: 94, + }, + { + jobId: 95, + }, + { + jobId: 96, + }, + { + jobId: 97, + }, + { + jobId: 98, + }, + { + jobId: 99, + }, + { + jobId: 100, + }, + { + jobId: 101, + }, + { + jobId: 102, + }, + { + jobId: 103, + }, + { + jobId: 104, + }, + { + jobId: 105, + }, + { + jobId: 106, + }, + { + jobId: 107, + }, + { + jobId: 108, + }, + { + jobId: 109, + }, + { + jobId: 110, + }, + { + jobId: 111, + }, + { + jobId: 112, + }, + { + jobId: 113, + }, + { + jobId: 114, + }, + { + jobId: 115, + }, + { + jobId: 116, + }, + { + jobId: 117, + }, + { + jobId: 118, + }, + { + jobId: 119, + }, + { + jobId: 120, + }, + { + jobId: 121, + }, + { + jobId: 122, + }, + { + jobId: 123, + }, + { + jobId: 124, + }, + { + jobId: 125, + }, + { + jobId: 126, + }, + { + jobId: 127, + }, + { + jobId: 128, + }, + { + jobId: 129, + }, + { + jobId: 130, + }, + { + jobId: 131, + }, + { + jobId: 132, + }, + { + jobId: 133, + }, + { + jobId: 134, + }, + { + jobId: 135, + }, + { + jobId: 136, + }, + { + jobId: 137, + }, + { + jobId: 138, + }, + { + jobId: 139, + }, + { + jobId: 140, + }, + { + jobId: 141, + }, + { + jobId: 142, + }, + { + jobId: 143, + }, + { + jobId: 144, + }, + { + jobId: 145, + }, + { + jobId: 146, + }, + { + jobId: 147, + }, + { + jobId: 148, + }, + { + jobId: 149, + }, + { + jobId: 150, + }, + { + jobId: 151, + }, + { + jobId: 152, + }, + { + jobId: 153, + }, + { + jobId: 154, + }, + { + jobId: 155, + }, + { + jobId: 156, + }, + { + jobId: 157, + }, + { + jobId: 158, + }, + { + jobId: 159, + }, + { + jobId: 160, + }, + { + jobId: 161, + }, + { + jobId: 162, + }, + { + jobId: 163, + }, + { + jobId: 164, + }, + { + jobId: 165, + }, + { + jobId: 166, + }, + { + jobId: 167, + }, + { + jobId: 168, + }, + { + jobId: 169, + }, + { + jobId: 170, + }, + { + jobId: 171, + }, + { + jobId: 172, + }, + { + jobId: 173, + }, + { + jobId: 174, + }, + { + jobId: 175, + }, + { + jobId: 176, + }, + { + jobId: 177, + }, + { + jobId: 178, + }, + { + jobId: 179, + }, + { + jobId: 180, + }, + { + jobId: 181, + }, + { + jobId: 182, + }, + { + jobId: 183, + }, + { + jobId: 184, + }, + { + jobId: 185, + }, + { + jobId: 186, + }, + { + jobId: 187, + }, + { + jobId: 188, + }, + { + jobId: 189, + }, + { + jobId: 190, + }, + { + jobId: 191, + }, + { + jobId: 192, + }, + { + jobId: 193, + }, + { + jobId: 194, + }, + { + jobId: 195, + }, + { + jobId: 196, + }, + { + jobId: 197, + }, + { + jobId: 198, + }, + { + jobId: 199, + }, + { + jobId: 200, + }, + { + jobId: 201, + }, + { + jobId: 202, + }, + { + jobId: 203, + }, + { + jobId: 204, + }, + { + jobId: 205, + }, + { + jobId: 206, + }, + { + jobId: 207, + }, + { + jobId: 208, + }, + { + jobId: 209, + }, + { + jobId: 210, + }, + { + jobId: 211, + }, + { + jobId: 212, + }, + { + jobId: 213, + }, + { + jobId: 214, + }, + { + jobId: 215, + }, + { + jobId: 216, + }, + { + jobId: 217, + }, + { + jobId: 218, + }, + { + jobId: 219, + }, + { + jobId: 220, + }, + { + jobId: 221, + }, + { + jobId: 222, + }, + { + jobId: 223, + }, + { + jobId: 224, + }, + { + jobId: 225, + }, + { + jobId: 226, + }, + { + jobId: 227, + }, + { + jobId: 228, + }, + { + jobId: 229, + }, + { + jobId: 230, + }, + { + jobId: 231, + }, + { + jobId: 232, + }, + { + jobId: 233, + }, + { + jobId: 234, + }, + { + jobId: 235, + }, + { + jobId: 236, + }, + { + jobId: 237, + }, + { + jobId: 238, + }, + { + jobId: 239, + }, + { + jobId: 240, + }, + { + jobId: 241, + }, + { + jobId: 242, + }, + { + jobId: 243, + }, + { + jobId: 244, + }, + { + jobId: 245, + }, + { + jobId: 246, + }, + { + jobId: 247, + }, + { + jobId: 248, + }, + { + jobId: 249, + }, + { + jobId: 250, + }, + { + jobId: 251, + }, + { + jobId: 252, + }, + { + jobId: 253, + }, + { + jobId: 254, + }, + { + jobId: 255, + }, + { + jobId: 256, + }, + { + jobId: 257, + }, + { + jobId: 258, + }, + { + jobId: 259, + }, + { + jobId: 260, + }, + { + jobId: 261, + }, + { + jobId: 262, + }, + { + jobId: 263, + }, + { + jobId: 264, + }, + { + jobId: 265, + }, + { + jobId: 266, + }, + { + jobId: 267, + }, + { + jobId: 268, + }, + { + jobId: 269, + }, + { + jobId: 270, + }, + { + jobId: 271, + }, + { + jobId: 272, + }, + { + jobId: 273, + }, + { + jobId: 274, + }, + { + jobId: 275, + }, + { + jobId: 276, + }, + { + jobId: 277, + }, + { + jobId: 278, + }, + { + jobId: 279, + }, + { + jobId: 280, + }, + { + jobId: 281, + }, + { + jobId: 282, + }, + { + jobId: 283, + }, + { + jobId: 284, + }, + { + jobId: 285, + }, + { + jobId: 286, + }, + { + jobId: 287, + }, + { + jobId: 288, + }, + { + jobId: 289, + }, + { + jobId: 290, + }, + { + jobId: 291, + }, + { + jobId: 292, + }, + { + jobId: 293, + }, + { + jobId: 294, + }, + { + jobId: 295, + }, + { + jobId: 296, + }, + { + jobId: 297, + }, + { + jobId: 298, + }, + { + jobId: 299, + }, + { + jobId: 300, + }, + { + jobId: 301, + }, + { + jobId: 302, + }, + { + jobId: 303, + }, + { + jobId: 304, + }, + { + jobId: 305, + }, + { + jobId: 306, + }, + { + jobId: 307, + }, + { + jobId: 308, + }, + { + jobId: 309, + }, + { + jobId: 310, + }, + { + jobId: 311, + }, + { + jobId: 312, + }, + { + jobId: 313, + }, + { + jobId: 314, + }, + { + jobId: 315, + }, + { + jobId: 316, + }, + { + jobId: 317, + }, + { + jobId: 318, + }, + { + jobId: 319, + }, + { + jobId: 320, + }, + { + jobId: 321, + }, + { + jobId: 322, + }, + { + jobId: 323, + }, + { + jobId: 324, + }, + { + jobId: 325, + }, + { + jobId: 326, + }, + { + jobId: 327, + }, + { + jobId: 328, + }, + { + jobId: 329, + }, + { + jobId: 330, + }, + { + jobId: 331, + }, + { + jobId: 332, + }, + { + jobId: 333, + }, + { + jobId: 334, + }, + { + jobId: 335, + }, + { + jobId: 336, + }, + { + jobId: 337, + }, + { + jobId: 338, + }, + { + jobId: 339, + }, + { + jobId: 340, + }, + { + jobId: 341, + }, + { + jobId: 342, + }, + { + jobId: 343, + }, + { + jobId: 344, + }, + { + jobId: 345, + }, + { + jobId: 346, + }, + { + jobId: 347, + }, + { + jobId: 348, + }, + { + jobId: 349, + }, + { + jobId: 350, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, +]; + +const mixedBatchOutput = [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=911&id=912&id=913&id=914&id=915&id=916&id=917&id=918&id=919&id=920', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 911, + }, + { + jobId: 912, + }, + { + jobId: 913, + }, + { + jobId: 914, + }, + { + jobId: 915, + }, + { + jobId: 916, + }, + { + jobId: 917, + }, + { + jobId: 918, + }, + { + jobId: 919, + }, + { + jobId: 920, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=901&id=902&id=903&id=904&id=905&id=906&id=907&id=908&id=909&id=910', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 901, + }, + { + jobId: 902, + }, + { + jobId: 903, + }, + { + jobId: 904, + }, + { + jobId: 905, + }, + { + jobId: 906, + }, + { + jobId: 907, + }, + { + jobId: 908, + }, + { + jobId: 909, + }, + { + jobId: 910, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=704&id=705&id=706', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=501&id=502&id=503', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 2, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, +]; + +const recordEventGenerator = (id, action, externalId) => { + // this function is used to generate record events for testing + const testRecordEvent = { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: DEST_DEFINITION, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action, + fields: { + id, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: externalId, + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: id, + }, + }; + return testRecordEvent; +}; + +module.exports = { + recordInputs, + audiencelistInputs, + reqMetadata, + recordOutput, + largeRecordOutput, + mixedBatchOutput, + recordEventGenerator, +}; diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index d70d2b5a2e..d9fd06d70e 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -4,19 +4,20 @@ const { defaultPostRequestConfig, defaultDeleteRequestConfig, generateErrorObject, + simpleProcessRouterDest, } = require('../../util'); const { AUTH_CACHE_TTL, JSON_MIME_TYPE } = require('../../util/constant'); -const { getIds, validateMessageType, transformForRecordEvent } = require('./util'); +const { getIds, validateMessageType } = require('./util'); const { getDestinationExternalID, defaultRequestConfig, getErrorRespEvents, - simpleProcessRouterDest, } = require('../../util'); const { formatConfig, MAX_LEAD_IDS_SIZE } = require('./config'); const Cache = require('../../util/cache'); const { getAuthToken } = require('../marketo/transform'); const { InstrumentationError, UnauthorizedError } = require('../../util/errorTypes'); +const { processRecordInputs } = require('./transformV2'); const authCache = new Cache(AUTH_CACHE_TTL); // 1 hr @@ -57,10 +58,10 @@ const batchResponseBuilder = (message, Config, token, leadIds, operation) => { return response; }; -const processEvent = (input) => { - const { token, message, destination } = input; +const processEvent = (event) => { + const { token, message, destination } = event; const { Config } = destination; - validateMessageType(message, ['audiencelist', 'record']); + validateMessageType(message, ['audiencelist']); const response = []; let toAdd; let toRemove; @@ -94,66 +95,71 @@ const processEvent = (input) => { return response; }; -const process = async (event) => { +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const process = async (event, _processParams) => { const token = await getAuthToken(formatConfig(event.destination)); - if (!token) { throw new UnauthorizedError('Authorization failed'); } - const response = processEvent({ ...event, token }); + const updatedEvent = { ...event, token }; + const response = processEvent(updatedEvent); return response; }; const processRouterDest = async (inputs, reqMetadata) => { // Token needs to be generated for marketo which will be done on input level. // If destination information is not present Error should be thrown - let token; + + const { destination } = inputs[0]; try { - token = await getAuthToken(formatConfig(inputs[0].destination)); + const token = await getAuthToken(formatConfig(destination)); if (!token) { throw new UnauthorizedError('Could not retrieve authorisation token'); } } catch (error) { - // Not using handleRtTfSingleEventError here as this is for multiple events - const errObj = generateErrorObject(error); - const respEvents = getErrorRespEvents( - inputs.map((input) => input.metadata), - errObj.status, - errObj.message, - errObj.statTags, + const errorObj = generateErrorObject(error); + const errResponses = inputs.map((input) => + getErrorRespEvents(input.metadata, errorObj.status, errorObj.message, errorObj.statTags), ); - return [{ ...respEvents, destination: inputs?.[0]?.destination }]; + + return errResponses; } - // Checking previous status Code. Initially setting to false. - // If true then previous status is 500 and every subsequent event output should be - // sent with status code 500 to the router to be retried. - const tokenisedInputs = inputs.map((input) => ({ ...input, token })); - const leadIdObj = { - insert: [], - delete: [], - }; // use lodash.groupby to group the inputs based on message type - let transformedRecordEvent = []; + const transformedRecordEvent = []; let transformedAudienceEvent = []; - const groupedInputs = lodash.groupBy(tokenisedInputs, (input) => input.message.type); + const groupedInputs = lodash.groupBy(inputs, (input) => input.message.type); - if (groupedInputs.record && groupedInputs.record.length > 0) { - const finalInputForRecordEvent = transformForRecordEvent(groupedInputs.record, leadIdObj); - transformedRecordEvent = await simpleProcessRouterDest( - finalInputForRecordEvent, - processEvent, - reqMetadata, + const respList = []; + // process record events + if (Array.isArray(groupedInputs.record) && groupedInputs.record.length > 0) { + const groupedRecordInputs = groupedInputs.record; + const { staticListId } = destination.Config; + const externalIdGroupedRecordInputs = lodash.groupBy( + groupedRecordInputs, + (input) => getDestinationExternalID(input.message, 'marketoStaticListId') || staticListId, + ); + const alltransformedGroupedRecordEvent = await Promise.all( + Object.keys(externalIdGroupedRecordInputs).map(async (key) => { + const transformedGroupedRecordEvent = await processRecordInputs( + externalIdGroupedRecordInputs[key], + destination, + ); + return transformedGroupedRecordEvent; + }), ); + + transformedRecordEvent.push(...alltransformedGroupedRecordEvent.flat()); } + // process audiencelist events if (groupedInputs.audiencelist && groupedInputs.audiencelist.length > 0) { transformedAudienceEvent = await simpleProcessRouterDest( groupedInputs.audiencelist, - processEvent, + process, reqMetadata, ); } - const respList = [...transformedRecordEvent, ...transformedAudienceEvent]; + respList.push(...transformedRecordEvent, ...transformedAudienceEvent); return respList; }; @@ -164,23 +170,19 @@ const processRouterDest = async (inputs, reqMetadata) => { */ function processMetadataForRouter(output) { const { metadata, destination } = output; - // check if metadata[0] is an array or not - let clonedMetadata; - if (Array.isArray(metadata[0])) { - [clonedMetadata] = cloneDeep(metadata); - } else { - clonedMetadata = cloneDeep(metadata); - } + const clonedMetadata = cloneDeep(metadata); clonedMetadata.forEach((metadataElement) => { // eslint-disable-next-line no-param-reassign - metadataElement.destInfo = { authKey: destination.ID }; + metadataElement.destInfo = { authKey: destination?.ID }; }); return clonedMetadata; } module.exports = { process, + processEvent, processRouterDest, processMetadataForRouter, authCache, + batchResponseBuilder, }; diff --git a/src/v0/destinations/marketo_static_list/transform.test.js b/src/v0/destinations/marketo_static_list/transform.test.js new file mode 100644 index 0000000000..d18737684a --- /dev/null +++ b/src/v0/destinations/marketo_static_list/transform.test.js @@ -0,0 +1,54 @@ +const { processRouterDest } = require('./transform'); +const axios = require('axios'); +const MockAxiosAdapter = require('axios-mock-adapter'); +const { + recordInputs, + audiencelistInputs, + reqMetadata, + recordOutput, + largeRecordOutput, + mixedBatchOutput, + recordEventGenerator, +} = require('./testData/testData'); + +const mockAdapter = new MockAxiosAdapter(axios, { onNoMatch: 'throwException' }); +beforeAll(() => { + mockAdapter + .onGet('https://marketo_acct_id_success.mktorest.com/identity/oauth/token') + .reply(200, { access_token: 'access_token_success' }); +}); +describe('Unit cases covering the router flow for both record and audiencelist event types', () => { + it('Sending a small batch of only record events (4 events)', async () => { + const inputs = recordInputs; + const result = await processRouterDest(inputs, reqMetadata); + expect(result).toEqual(recordOutput); + }); + + it('Sending a large batch of only record events (450 events: 350 inserts | 100 deletes )', async () => { + const largeRecordInputs = []; + for (let index = 0; index < 350; index++) { + largeRecordInputs.push(recordEventGenerator(index + 1, 'insert', 1122)); + } + for (let index = 350; index < 450; index++) { + largeRecordInputs.push(recordEventGenerator(index + 1, 'delete', 1122)); + } + const result = await processRouterDest(largeRecordInputs, reqMetadata); + expect(result).toEqual(largeRecordOutput); + }); + + it('Sending a mixed batch of record and audiencelist events (22 events: 10 inserts | 10 deletes | 2 audiencelist (⌐■_■) )', async () => { + const mixedBatchInputs = []; + for (let index = 900; index < 910; index++) { + mixedBatchInputs.push(recordEventGenerator(index + 1, 'insert', 1122)); + } + for (let index = 910; index < 920; index++) { + mixedBatchInputs.push(recordEventGenerator(index + 1, 'delete', 1122)); + } + mixedBatchInputs.push(...audiencelistInputs); + const result = await processRouterDest(mixedBatchInputs, reqMetadata); + expect(result).toEqual(mixedBatchOutput); + }); +}); +afterAll(() => { + mockAdapter.restore(); +}); diff --git a/src/v0/destinations/marketo_static_list/transformV2.js b/src/v0/destinations/marketo_static_list/transformV2.js new file mode 100644 index 0000000000..c14204cbfa --- /dev/null +++ b/src/v0/destinations/marketo_static_list/transformV2.js @@ -0,0 +1,135 @@ +const lodash = require('lodash'); +const { + defaultPostRequestConfig, + defaultDeleteRequestConfig, + getDestinationExternalID, + defaultRequestConfig, + getSuccessRespEvents, + isDefinedAndNotNull, + generateErrorObject, + getErrorRespEvents, +} = require('../../util'); +const { JSON_MIME_TYPE } = require('../../util/constant'); +const { MAX_LEAD_IDS_SIZE } = require('./config'); +const { InstrumentationError, UnauthorizedError } = require('../../util/errorTypes'); +const { getAuthToken } = require('../marketo/transform'); +const { formatConfig } = require('../marketo/config'); + +/** + * Generates the final response structure to be sent to the destination + * @param {*} endPoint + * @param {*} leadIds + * @param {*} operation + * @param {*} token + * @returns batched response + */ +const responseBuilder = (endPoint, leadIds, operation, token) => { + let updatedEndpoint = endPoint; + if (leadIds.length > 0) { + leadIds.forEach((id) => { + updatedEndpoint = `${updatedEndpoint}id=${id}&`; + }); + } + updatedEndpoint = updatedEndpoint.slice(0, -1); + const response = defaultRequestConfig(); + response.endpoint = updatedEndpoint; + if (operation === 'insert') { + response.method = defaultPostRequestConfig.requestMethod; + } else { + response.method = defaultDeleteRequestConfig.requestMethod; + } + response.headers = { + Authorization: `Bearer ${token}`, + 'Content-Type': JSON_MIME_TYPE, + }; + return response; +}; + +/** + * The function is responsible for building the batched response for a given set of record inputs. + * @param {*} groupedRecordInputs + * @param {*} Config + * @param {*} token + * @param {*} leadIds + * @param {*} operation + * @returns an array of response objects, where each object represents a batched response for a chunk of lead IDs. + */ +const batchResponseBuilder = (listId, Config, token, leadIds, operation) => { + const { accountId } = Config; + const endpoint = `https://${accountId}.mktorest.com/rest/v1/lists/${listId}/leads.json?`; + const response = []; + const leadIdsChunks = lodash.chunk(leadIds, MAX_LEAD_IDS_SIZE); + leadIdsChunks.forEach((ids) => { + response.push(responseBuilder(endpoint, ids, operation, token)); + }); + return response; +}; + +/** + * A function that processes a list of grouped record inputs. + * It iterates through each input and groups the field IDs based on the action. + * @param {*} groupedRecordInputs + * @returns An array containing the batched responses for the insert and delete actions along with the metadata. + */ +async function processRecordInputs(groupedRecordInputs, destination) { + const token = await getAuthToken(formatConfig(destination)); + if (!token) { + throw new UnauthorizedError('Authorization failed'); + } + const { Config } = destination; + const listId = + getDestinationExternalID(groupedRecordInputs[0].message, 'marketoStaticListId') || + Config.staticListId; + + // iterate through each input and group field id based on action + const insertFields = []; + const deleteFields = []; + const successMetadataForInsert = []; + const successMetadataForDelete = []; + const errorMetadata = []; + + groupedRecordInputs.forEach((input) => { + const { fields, action } = input.message; + const fieldId = fields.id; + if (action === 'insert' && isDefinedAndNotNull(fieldId)) { + insertFields.push(fieldId); + successMetadataForInsert.push(input.metadata); + } else if (action === 'delete' && isDefinedAndNotNull(fieldId)) { + deleteFields.push(fieldId); + successMetadataForDelete.push(input.metadata); + } else { + errorMetadata.push(input.metadata); + } + }); + const deletePayloads = batchResponseBuilder(listId, Config, token, deleteFields, 'delete'); + + const deleteResponse = getSuccessRespEvents( + deletePayloads, + successMetadataForDelete, + destination, + true, + ); + + const insertPayloads = batchResponseBuilder(listId, Config, token, insertFields, 'insert'); + + const insertResponse = getSuccessRespEvents( + insertPayloads, + successMetadataForInsert, + destination, + true, + ); + + const error = new InstrumentationError( + 'Invalid leadIds format or no leadIds found neither to add nor to remove', + ); + const errorObj = generateErrorObject(error); + const errorResponseList = errorMetadata.map((metadata) => + getErrorRespEvents(metadata, errorObj.status, errorObj.message, errorObj.statTags), + ); + + return [deleteResponse, insertResponse, ...errorResponseList]; +} + +module.exports = { + processRecordInputs, +}; diff --git a/src/v0/destinations/marketo_static_list/util.js b/src/v0/destinations/marketo_static_list/util.js index 7fd20a56d6..83ca27f548 100644 --- a/src/v0/destinations/marketo_static_list/util.js +++ b/src/v0/destinations/marketo_static_list/util.js @@ -35,47 +35,7 @@ const validateMessageType = (message, allowedTypes) => { } }; -function transformForRecordEvent(inputs, leadIdObj) { - const finalMetadata = []; - // iterate through each inputs metadata and create a final metadata - const tokenisedInputs = inputs.map((input) => { - const { message } = input; - const { metadata } = input; - finalMetadata.push(metadata); - const { fields, action, type } = message; - if (type !== 'record') { - throw new InstrumentationError('Invalid message type, Supported message type is record.'); - } - const { properties } = message; - if (!properties) { - message.properties = {}; - } - message.properties.listData = message?.properties.listData || { add: [], remove: [] }; - const fieldsId = fields?.id; - if (fieldsId === undefined) { - throw new InstrumentationError('No lead id passed in the payload.'); - } - if (action === 'insert') { - leadIdObj.insert.push({ id: fieldsId }); - message.properties.listData.add.push({ id: fieldsId }); - } else if (action === 'delete') { - leadIdObj.delete.push({ id: fieldsId }); - message.properties.listData.remove.push({ id: fieldsId }); - } else { - throw new InstrumentationError('Invalid action type'); - } - return input; - }); - const finalInput = [tokenisedInputs[0]]; - finalInput[0].metadata = finalMetadata; - finalInput[0].message.properties.listData.add = leadIdObj.insert; - finalInput[0].message.properties.listData.remove = leadIdObj.delete; - - return finalInput; -} - module.exports = { getIds, validateMessageType, - transformForRecordEvent, }; diff --git a/src/v0/destinations/marketo_static_list/util.test.js b/src/v0/destinations/marketo_static_list/util.test.js deleted file mode 100644 index c9b1f803a5..0000000000 --- a/src/v0/destinations/marketo_static_list/util.test.js +++ /dev/null @@ -1,243 +0,0 @@ -const { InstrumentationError } = require('../../util/errorTypes'); -const { - transformForRecordEvent, -} = require('../../../../src/v0/destinations/marketo_static_list/util'); - -describe('transformForRecordEvent', () => { - // Should correctly transform an input with action 'insert' - test('should correctly transform an input with action "insert"', () => { - const inputs = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'insert', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata1', - }, - ]; - const leadIdObj = { insert: [], delete: [] }; - const expectedOutput = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'insert', - properties: { listData: { add: [{ id: 1 }], remove: [] } }, - }, - metadata: ['metadata1'], - }, - ]; - - const result = transformForRecordEvent(inputs, leadIdObj); - - expect(result).toEqual(expectedOutput); - }); - - // Should correctly transform an input with action 'delete' - test('should correctly transform an input with action "delete"', () => { - const inputs = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'delete', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata1', - }, - ]; - const leadIdObj = { insert: [], delete: [] }; - const expectedOutput = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'delete', - properties: { listData: { add: [], remove: [{ id: 1 }] } }, - }, - metadata: ['metadata1'], - }, - ]; - - const result = transformForRecordEvent(inputs, leadIdObj); - - expect(result).toEqual(expectedOutput); - }); - - // Should raise an InstrumentationError if action is not valid - test('should raise an InstrumentationError if action is not valid', () => { - const inputs = [ - { - message: { - fields: { id: 1 }, - action: 'invalid', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata1', - }, - ]; - const leadIdObj = { insert: [], delete: [] }; - - expect(() => { - transformForRecordEvent(inputs, leadIdObj); - }).toThrow(InstrumentationError); - }); - - // Should raise an InstrumentationError on input with no fields - test('should raise an InstrumentationError on input with no fields', () => { - const inputs = [ - { - message: { - action: 'insert', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata1', - }, - ]; - const leadIdObj = { insert: [], delete: [] }; - - expect(() => { - transformForRecordEvent(inputs, leadIdObj); - }).toThrow(InstrumentationError); - }); - - // Should correctly transform an input with 2 insert payloads - test('should correctly transform an input with 2 insert payloads', () => { - const inputs = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'insert', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata1', - }, - { - message: { - type: 'record', - fields: { id: 2 }, - action: 'insert', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata2', - }, - ]; - const leadIdObj = { insert: [], delete: [] }; - const expectedOutput = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'insert', - properties: { listData: { add: [{ id: 1 }, { id: 2 }], remove: [] } }, - }, - metadata: ['metadata1', 'metadata2'], - }, - ]; - - const result = transformForRecordEvent(inputs, leadIdObj); - - expect(result).toEqual(expectedOutput); - }); - - // Should correctly transform an input with 2 delete payloads - test('should correctly transform an input with 2 delete payloads', () => { - const inputs = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'delete', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata1', - }, - { - message: { - type: 'record', - fields: { id: 2 }, - action: 'delete', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata2', - }, - ]; - const leadIdObj = { insert: [], delete: [] }; - const expectedOutput = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'delete', - properties: { listData: { remove: [{ id: 1 }, { id: 2 }], add: [] } }, - }, - metadata: ['metadata1', 'metadata2'], - }, - ]; - - const result = transformForRecordEvent(inputs, leadIdObj); - - expect(result).toEqual(expectedOutput); - }); - - // Should correctly transform an input with 2 insert and 2 delete payloads - test('should correctly transform an input with 2 insert and 2 delete payloads', () => { - const inputs = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'delete', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata1', - }, - { - message: { - type: 'record', - fields: { id: 2 }, - action: 'insert', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata2', - }, - { - message: { - type: 'record', - fields: { id: 3 }, - action: 'delete', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata3', - }, - { - message: { - type: 'record', - fields: { id: 4 }, - action: 'insert', - properties: { listData: { add: [], remove: [] } }, - }, - metadata: 'metadata4', - }, - ]; - const leadIdObj = { insert: [], delete: [] }; - const expectedOutput = [ - { - message: { - type: 'record', - fields: { id: 1 }, - action: 'delete', - properties: { listData: { remove: [{ id: 1 }, { id: 3 }], add: [{ id: 2 }, { id: 4 }] } }, - }, - metadata: ['metadata1', 'metadata2', 'metadata3', 'metadata4'], - }, - ]; - - const result = transformForRecordEvent(inputs, leadIdObj); - - expect(result).toEqual(expectedOutput); - }); -}); diff --git a/test/integrations/destinations/marketo_static_list/router/data.ts b/test/integrations/destinations/marketo_static_list/router/data.ts index abf37c3294..fa97ba49f2 100644 --- a/test/integrations/destinations/marketo_static_list/router/data.ts +++ b/test/integrations/destinations/marketo_static_list/router/data.ts @@ -1,7 +1,7 @@ export const data = [ { name: 'marketo_static_list', - description: 'Test 0', + description: 'Test 0: Test audiencelist event with add and remove', feature: 'router', module: 'destination', version: 'v0', @@ -1334,7 +1334,7 @@ export const data = [ }, { name: 'marketo_static_list', - description: 'Test 1', + description: 'Test 1: Test record and audiencelist events in a single batch', feature: 'router', module: 'destination', version: 'v0', @@ -1379,7 +1379,7 @@ export const data = [ }, externalId: [ { - type: 'MARKETO_STATIC_LIST-leadId', + type: 'marketoStaticListId', identifierType: 'id', }, ], @@ -1429,7 +1429,7 @@ export const data = [ }, externalId: [ { - type: 'MARKETO_STATIC_LIST-leadId', + type: 'marketoStaticListId', identifierType: 'id', }, ], @@ -1479,7 +1479,7 @@ export const data = [ }, externalId: [ { - type: 'MARKETO_STATIC_LIST-leadId', + type: 'marketoStaticListId', identifierType: 'id', }, ], @@ -1520,7 +1520,7 @@ export const data = [ externalId: [ { identifierType: 'id', - type: 'MARKETO_STATIC_LIST-leadId', + type: 'marketoStaticListId', }, ], mappedToDestination: 'true', @@ -1570,7 +1570,7 @@ export const data = [ externalId: [ { identifierType: 'id', - type: 'MARKETO_STATIC_LIST-leadId', + type: 'marketoStaticListId', }, ], mappedToDestination: 'true', @@ -1620,7 +1620,7 @@ export const data = [ externalId: [ { identifierType: 'id', - type: 'MARKETO_STATIC_LIST-leadId', + type: 'marketoStaticListId', }, ], mappedToDestination: 'true', @@ -2837,6 +2837,52 @@ export const data = [ }, files: {}, }, + ], + metadata: [ + { + jobId: 4, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 5, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 6, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ { version: '1', type: 'REST', @@ -2876,26 +2922,8 @@ export const data = [ authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', }, }, - { - jobId: 4, - destInfo: { - authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', - }, - }, - { - jobId: 5, - destInfo: { - authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', - }, - }, - { - jobId: 6, - destInfo: { - authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', - }, - }, ], - batched: false, + batched: true, statusCode: 200, destination: { ID: '1zwa1wKshSt81YksKmUdJnr4IOK', @@ -3067,4 +3095,554 @@ export const data = [ }, }, }, + { + name: 'marketo_static_list', + description: 'Test 2: Test record evemts', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + destType: 'marketo_static_list', + input: [ + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1001, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 1, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1002, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + id: 'id002', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 2, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1003, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 3, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id002', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2001', + }, + type: 'record', + }, + metadata: { + jobId: 4, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id001', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2002', + }, + type: 'record', + }, + metadata: { + jobId: 5, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id002', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2003', + }, + type: 'record', + }, + metadata: { + jobId: 6, + }, + }, + ], + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=2002', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 5, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=1001&id=1003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 3, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=2001&id=2003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 4, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 6, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=1002', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 2, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + ], + }, + }, + }, + }, ]; From 67ab8456fe9c0ef890082fb3e9e9964b51f43207 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Wed, 22 Nov 2023 22:25:35 +0530 Subject: [PATCH 10/15] fix: unit case with empty request --- .../marketo_static_list/testData/constants.js | 34 ++++++++----------- .../marketo_static_list/testData/testData.js | 7 ---- .../marketo_static_list/transformV2.js | 14 ++++++-- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/testData/constants.js b/src/v0/destinations/marketo_static_list/testData/constants.js index 0ba27c277d..2c420c6bc2 100644 --- a/src/v0/destinations/marketo_static_list/testData/constants.js +++ b/src/v0/destinations/marketo_static_list/testData/constants.js @@ -1,16 +1,23 @@ const EXTERNAL_ID = 'marketoStaticListId'; const TOKEN = 'Bearer access_token_success'; const CONTENT_TYPE = 'application/json'; +const DEST_CONFIG = { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, +}; +const DEST_DEFINITION = { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', +}; const DEST_OBJECT = { ID: '1zwa1wKshSt81YksKmUdJnr4IOK', Name: 'test_marketo_rc', - DestinationDefinition: { - ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', - Name: 'MARKETO', - DisplayName: 'Marketo', - transformAt: 'processor', - transformAtV1: 'processor', - }, + DestinationDefinition: DEST_DEFINITION, Config: { clientId: 'marketo_client_id_success', clientSecret: 'marketo_client_secret_success', @@ -21,24 +28,11 @@ const DEST_OBJECT = { Transformations: [], IsProcessorEnabled: true, }; -const DEST_DEFINITION = { - ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', - Name: 'MARKETO', - DisplayName: 'Marketo', - transformAt: 'processor', - transformAtV1: 'processor', -}; const MESSAGE_SOURCES_CONTEXT = { job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', job_run_id: 'ck99nbd2kqiljdihhkh0', task_run_id: 'ck99nbd2kqiljdihhkhg', }; -const DEST_CONFIG = { - clientId: 'marketo_client_id_success', - clientSecret: 'marketo_client_secret_success', - accountId: 'marketo_acct_id_success', - staticListId: 1234, -}; module.exports = { EXTERNAL_ID, diff --git a/src/v0/destinations/marketo_static_list/testData/testData.js b/src/v0/destinations/marketo_static_list/testData/testData.js index d0d5f24cb5..45328436ce 100644 --- a/src/v0/destinations/marketo_static_list/testData/testData.js +++ b/src/v0/destinations/marketo_static_list/testData/testData.js @@ -121,13 +121,6 @@ const recordInputs = [ ]; const recordOutput = [ - { - batchedRequest: [], - metadata: [], - batched: true, - statusCode: 200, - destination: DEST_OBJECT, - }, { batchedRequest: [ { diff --git a/src/v0/destinations/marketo_static_list/transformV2.js b/src/v0/destinations/marketo_static_list/transformV2.js index c14204cbfa..50b1b7cb09 100644 --- a/src/v0/destinations/marketo_static_list/transformV2.js +++ b/src/v0/destinations/marketo_static_list/transformV2.js @@ -126,8 +126,18 @@ async function processRecordInputs(groupedRecordInputs, destination) { const errorResponseList = errorMetadata.map((metadata) => getErrorRespEvents(metadata, errorObj.status, errorObj.message, errorObj.statTags), ); - - return [deleteResponse, insertResponse, ...errorResponseList]; + const finalResponse = []; + if (deleteResponse.batchedRequest.length > 0) { + finalResponse.push(deleteResponse); + } + if (insertResponse.batchedRequest.length > 0) { + finalResponse.push(insertResponse); + } + if (errorResponseList.length > 0) { + finalResponse.push(...errorResponseList); + } + return finalResponse; + // return [deleteResponse, insertResponse, ...errorResponseList]; } module.exports = { From 15c9b5ced83bc154f60c75be60d3f3bd2e3330af Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:31:41 +0530 Subject: [PATCH 11/15] fix: missed error lib --- src/v0/destinations/marketo_static_list/transformV2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v0/destinations/marketo_static_list/transformV2.js b/src/v0/destinations/marketo_static_list/transformV2.js index 50b1b7cb09..553c935702 100644 --- a/src/v0/destinations/marketo_static_list/transformV2.js +++ b/src/v0/destinations/marketo_static_list/transformV2.js @@ -1,4 +1,5 @@ const lodash = require('lodash'); +const { InstrumentationError, UnauthorizedError } = require('@rudderstack/integrations-lib'); const { defaultPostRequestConfig, defaultDeleteRequestConfig, @@ -11,7 +12,6 @@ const { } = require('../../util'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { MAX_LEAD_IDS_SIZE } = require('./config'); -const { InstrumentationError, UnauthorizedError } = require('../../util/errorTypes'); const { getAuthToken } = require('../marketo/transform'); const { formatConfig } = require('../marketo/config'); From ab61fa8de6d793ada4dde65b32c1afdffedc6a9e Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Thu, 23 Nov 2023 13:04:56 +0530 Subject: [PATCH 12/15] chore: refactor tests --- .../marketo_static_list/transform.test.js | 90 ++++++++++++++++++- 1 file changed, 87 insertions(+), 3 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/transform.test.js b/src/v0/destinations/marketo_static_list/transform.test.js index d18737684a..3c387c4295 100644 --- a/src/v0/destinations/marketo_static_list/transform.test.js +++ b/src/v0/destinations/marketo_static_list/transform.test.js @@ -21,7 +21,26 @@ describe('Unit cases covering the router flow for both record and audiencelist e it('Sending a small batch of only record events (4 events)', async () => { const inputs = recordInputs; const result = await processRouterDest(inputs, reqMetadata); - expect(result).toEqual(recordOutput); + + // assert that the result is as expected + expect(result.length).toEqual(3); + expect(result[0].batchedRequest.length).toEqual(1); // 1 batched request for 2 record events + expect(result[1].batchedRequest.length).toEqual(1); // 1 batched request for 1 record events + expect(result[2].batchedRequest.length).toEqual(1); // 1 batched request for 1 record events + expect(result[0].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=1001&id=1003', + ); // 1 api call for 2 leadIds + expect(result[1].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=2001', + ); // 1 api call for 1 leadId + expect(result[2].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=1002', + ); // 1 api call for 1 leadId + expect(result[1].batchedRequest[0].method).toEqual('DELETE'); // DELETE requests are sent first + expect(result[0].metadata.length).toEqual(2); // 2 metadata objects for 2 record events + expect(result[1].metadata.length).toEqual(1); // 1 metadata object for 1 record event + expect(result[2].metadata.length).toEqual(1); // 1 metadata object for 1 record event + expect(result).toEqual(recordOutput); // overall result should be equal to the expected output }); it('Sending a large batch of only record events (450 events: 350 inserts | 100 deletes )', async () => { @@ -33,7 +52,32 @@ describe('Unit cases covering the router flow for both record and audiencelist e largeRecordInputs.push(recordEventGenerator(index + 1, 'delete', 1122)); } const result = await processRouterDest(largeRecordInputs, reqMetadata); - expect(result).toEqual(largeRecordOutput); + + // assert that the result is as expected + /* + Total 3 API calls + 1. 1 API call for 100 DELETE requests + 2. 1 API call for 100 POST requests = limit reached for Marketo, leads split to next API call + 3. 1 API call for 50 POST requests + */ + expect(result.length).toEqual(2); + expect(result[0].batchedRequest.length).toEqual(1); // 1 batched request for 1 record event + expect(result[1].batchedRequest.length).toEqual(2); // 1 batched request for 2 record events + expect(result[0].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=351&id=352&id=353&id=354&id=355&id=356&id=357&id=358&id=359&id=360&id=361&id=362&id=363&id=364&id=365&id=366&id=367&id=368&id=369&id=370&id=371&id=372&id=373&id=374&id=375&id=376&id=377&id=378&id=379&id=380&id=381&id=382&id=383&id=384&id=385&id=386&id=387&id=388&id=389&id=390&id=391&id=392&id=393&id=394&id=395&id=396&id=397&id=398&id=399&id=400&id=401&id=402&id=403&id=404&id=405&id=406&id=407&id=408&id=409&id=410&id=411&id=412&id=413&id=414&id=415&id=416&id=417&id=418&id=419&id=420&id=421&id=422&id=423&id=424&id=425&id=426&id=427&id=428&id=429&id=430&id=431&id=432&id=433&id=434&id=435&id=436&id=437&id=438&id=439&id=440&id=441&id=442&id=443&id=444&id=445&id=446&id=447&id=448&id=449&id=450', + ); + expect(result[1].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299&id=300', + ); + expect(result[1].batchedRequest[1].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + ); + expect(result[0].batchedRequest[0].method).toEqual('DELETE'); // DELETE requests are sent first + expect(result[1].batchedRequest[0].method).toEqual('POST'); + expect(result[1].batchedRequest[1].method).toEqual('POST'); + expect(result[0].metadata.length).toEqual(100); // 100 metadata objects for 100 record events + expect(result[1].metadata.length).toEqual(350); // 350 metadata objects for 350 record events + expect(result).toEqual(largeRecordOutput); // overall result should be equal to the expected output }); it('Sending a mixed batch of record and audiencelist events (22 events: 10 inserts | 10 deletes | 2 audiencelist (⌐■_■) )', async () => { @@ -46,7 +90,47 @@ describe('Unit cases covering the router flow for both record and audiencelist e } mixedBatchInputs.push(...audiencelistInputs); const result = await processRouterDest(mixedBatchInputs, reqMetadata); - expect(result).toEqual(mixedBatchOutput); + + // assert that the result is as expected + /* + Total 4 API calls + 1. 1 API call for 10 DELETE requests + 2. 1 API call for 10 POST requests = limit reached for Marketo, leads split to next API call + 3. 1 API call for 2 POST requests + 4. 1 API call for 2 POST requests + */ + expect(result.length).toEqual(4); + expect(result[0].batchedRequest.length).toEqual(1); // 1 batched request for 1 record event + expect(result[1].batchedRequest.length).toEqual(1); // 1 batched request for 1 record event + expect(result[2].batchedRequest.length).toEqual(2); // 1 batched request for 2 audiencelist events + expect(result[3].batchedRequest.length).toEqual(2); // 1 batched request for 2 audiencelist events + expect(result[0].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=911&id=912&id=913&id=914&id=915&id=916&id=917&id=918&id=919&id=920', + ); + expect(result[1].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=901&id=902&id=903&id=904&id=905&id=906&id=907&id=908&id=909&id=910', + ); + expect(result[2].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=704&id=705&id=706', + ); + expect(result[2].batchedRequest[1].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=501&id=502&id=503', + ); + expect(result[3].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299', + ); + expect(result[2].batchedRequest[1].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=501&id=502&id=503', + ); + expect(result[0].batchedRequest[0].method).toEqual('DELETE'); // DELETE requests are sent first + expect(result[1].batchedRequest[0].method).toEqual('POST'); + expect(result[1].batchedRequest[0].method).toEqual('POST'); + expect(result[2].batchedRequest[0].method).toEqual('DELETE'); + expect(result[0].metadata.length).toEqual(10); + expect(result[1].metadata.length).toEqual(10); + expect(result[2].metadata.length).toEqual(1); + expect(result[3].metadata.length).toEqual(1); + expect(result).toEqual(mixedBatchOutput); // overall result should be equal to the expected output }); }); afterAll(() => { From 21ac7269c34d848558ba7fac75ba79735cb2e76f Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 28 Nov 2023 09:55:31 +0530 Subject: [PATCH 13/15] refactor: address comments --- .../marketo_static_list/transform.js | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 8d7e5af347..9194844db5 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -71,26 +71,22 @@ const processEvent = (event) => { if (message.properties?.listData?.remove) { toRemove = getIds(message.properties.listData.remove); } - if ( - (!Array.isArray(toAdd) || toAdd.length === 0) && - (!Array.isArray(toRemove) || toRemove.length === 0) - ) { + if (Array.isArray(toRemove) && toRemove.length > 0) { + const payload = batchResponseBuilder(message, Config, token, toRemove, 'remove'); + if (payload) { + response.push(...payload); + } + } + if (Array.isArray(toAdd) && toAdd.length > 0) { + const payload = batchResponseBuilder(message, Config, token, toAdd, 'add'); + if (payload) { + response.push(...payload); + } + } + if (response.length === 0) { throw new InstrumentationError( 'Invalid leadIds format or no leadIds found neither to add nor to remove', ); - } else { - if (Array.isArray(toRemove) && toRemove.length > 0) { - const payload = batchResponseBuilder(message, Config, token, toRemove, 'remove'); - if (payload) { - response.push(...payload); - } - } - if (Array.isArray(toAdd) && toAdd.length > 0) { - const payload = batchResponseBuilder(message, Config, token, toAdd, 'add'); - if (payload) { - response.push(...payload); - } - } } return response; }; From 9e3694bcb5a70c98d131efe6ce605c9df2d8c67a Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:41:12 +0530 Subject: [PATCH 14/15] chore: address commentsx2 --- src/v0/destinations/marketo_static_list/transform.js | 5 +++-- src/v0/destinations/marketo_static_list/transformV2.js | 8 +++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 9194844db5..f5c58043b8 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -128,7 +128,7 @@ const processRouterDest = async (inputs, reqMetadata) => { const respList = []; // process record events - if (Array.isArray(groupedInputs.record) && groupedInputs.record.length > 0) { + if (groupedInputs.record) { const groupedRecordInputs = groupedInputs.record; const { staticListId } = destination.Config; const externalIdGroupedRecordInputs = lodash.groupBy( @@ -140,6 +140,7 @@ const processRouterDest = async (inputs, reqMetadata) => { const transformedGroupedRecordEvent = await processRecordInputs( externalIdGroupedRecordInputs[key], destination, + key, ); return transformedGroupedRecordEvent; }), @@ -148,7 +149,7 @@ const processRouterDest = async (inputs, reqMetadata) => { transformedRecordEvent.push(...alltransformedGroupedRecordEvent.flat()); } // process audiencelist events - if (groupedInputs.audiencelist && groupedInputs.audiencelist.length > 0) { + if (groupedInputs.audiencelist) { transformedAudienceEvent = await simpleProcessRouterDest( groupedInputs.audiencelist, process, diff --git a/src/v0/destinations/marketo_static_list/transformV2.js b/src/v0/destinations/marketo_static_list/transformV2.js index 553c935702..9d37d5ed83 100644 --- a/src/v0/destinations/marketo_static_list/transformV2.js +++ b/src/v0/destinations/marketo_static_list/transformV2.js @@ -3,7 +3,6 @@ const { InstrumentationError, UnauthorizedError } = require('@rudderstack/integr const { defaultPostRequestConfig, defaultDeleteRequestConfig, - getDestinationExternalID, defaultRequestConfig, getSuccessRespEvents, isDefinedAndNotNull, @@ -69,17 +68,16 @@ const batchResponseBuilder = (listId, Config, token, leadIds, operation) => { * A function that processes a list of grouped record inputs. * It iterates through each input and groups the field IDs based on the action. * @param {*} groupedRecordInputs + * @param {*} destination + * @param {*} listId * @returns An array containing the batched responses for the insert and delete actions along with the metadata. */ -async function processRecordInputs(groupedRecordInputs, destination) { +async function processRecordInputs(groupedRecordInputs, destination, listId) { const token = await getAuthToken(formatConfig(destination)); if (!token) { throw new UnauthorizedError('Authorization failed'); } const { Config } = destination; - const listId = - getDestinationExternalID(groupedRecordInputs[0].message, 'marketoStaticListId') || - Config.staticListId; // iterate through each input and group field id based on action const insertFields = []; From e3c7950ea24bff94203df21d44174c5abcc2c9bc Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Wed, 6 Dec 2023 19:15:08 +0530 Subject: [PATCH 15/15] fix: error message, handle metadata, missing id --- src/v0/destinations/marketo_static_list/transform.js | 10 ++++++---- src/v0/destinations/marketo_static_list/transformV2.js | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index f5c58043b8..cbdb89f532 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -168,10 +168,12 @@ const processRouterDest = async (inputs, reqMetadata) => { function processMetadataForRouter(output) { const { metadata, destination } = output; const clonedMetadata = cloneDeep(metadata); - clonedMetadata.forEach((metadataElement) => { - // eslint-disable-next-line no-param-reassign - metadataElement.destInfo = { authKey: destination?.ID }; - }); + if (Array.isArray(clonedMetadata)) { + clonedMetadata.forEach((metadataElement) => { + // eslint-disable-next-line no-param-reassign + metadataElement.destInfo = { authKey: destination?.ID }; + }); + } return clonedMetadata; } diff --git a/src/v0/destinations/marketo_static_list/transformV2.js b/src/v0/destinations/marketo_static_list/transformV2.js index 9d37d5ed83..912d548d09 100644 --- a/src/v0/destinations/marketo_static_list/transformV2.js +++ b/src/v0/destinations/marketo_static_list/transformV2.js @@ -88,7 +88,7 @@ async function processRecordInputs(groupedRecordInputs, destination, listId) { groupedRecordInputs.forEach((input) => { const { fields, action } = input.message; - const fieldId = fields.id; + const fieldId = fields?.id; if (action === 'insert' && isDefinedAndNotNull(fieldId)) { insertFields.push(fieldId); successMetadataForInsert.push(input.metadata); @@ -118,7 +118,7 @@ async function processRecordInputs(groupedRecordInputs, destination, listId) { ); const error = new InstrumentationError( - 'Invalid leadIds format or no leadIds found neither to add nor to remove', + 'Invalid action type or no leadIds found neither to add nor to remove', ); const errorObj = generateErrorObject(error); const errorResponseList = errorMetadata.map((metadata) =>