Skip to content

Commit

Permalink
Merge branch 'develop' into fix.mailJetSource
Browse files Browse the repository at this point in the history
  • Loading branch information
anantjain45823 authored Jan 19, 2024
2 parents f8ed44a + 95f87e9 commit 7b80f4e
Show file tree
Hide file tree
Showing 31 changed files with 7,571 additions and 2,770 deletions.
68 changes: 68 additions & 0 deletions .github/workflows/component-test-report.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: Component Test Reporter

on:
pull_request:
types:
- opened
- reopened
- synchronize

permissions:
id-token: write # allows the JWT to be requested from GitHub's OIDC provider
contents: read # This is required for actions/checkout

jobs:
test_and_upload:
runs-on: ubuntu-latest

steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::${{ secrets.AWS_DEV_ACCOUNT_ID }}:role/${{ secrets.AWS_DEV_S3_SYNC_ROLE }}
aws-region: us-east-1

- name: Checkout
uses: actions/[email protected]
with:
fetch-depth: 1

- name: Setup Node
uses: actions/[email protected]
with:
node-version-file: '.nvmrc'
cache: 'npm'

- name: Install Dependencies
run: npm ci

- name: Run Tests and Generate Report
run: |
npm run test:ts -- component
- name: Uplaod Report to S3
run: |
aws s3 cp ./test_reports/ s3://test-integrations-dev/integrations-test-reports/rudder-transformer/${{ github.event.number }}/ --recursive
- name: Comment on PR with S3 Object URL
uses: actions/github-script@v7
with:
github-token: ${{ secrets.PAT }}
script: |
const { owner, repo } = context.repo;
// Get the pull request number
const prNumber = context.payload.pull_request.number;
const commentBody = `Test report for this run is available at: https://test-integrations-dev.s3.amazonaws.com/integrations-test-reports/rudder-transformer/${prNumber}/test-report.html`;
// Comment on the pull request
github.rest.issues.createComment({
owner,
repo,
issue_number: prNumber,
body: commentBody
});
2 changes: 1 addition & 1 deletion .github/workflows/prepare-for-dev-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
report-coverage:
name: Report Code Coverage
if: github.event_name == 'push'
uses: ./.github/workflows/report-code-coverage.yml
uses: ./.github/workflows/dt-test-and-report-code-coverage.yml
secrets:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/prepare-for-prod-dt-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
report-coverage:
name: Report Code Coverage
if: github.event_name == 'push'
uses: ./.github/workflows/report-code-coverage.yml
uses: ./.github/workflows/dt-test-and-report-code-coverage.yml
secrets:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/prepare-for-prod-ut-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
report-coverage:
name: Report Code Coverage
if: github.event_name == 'push'
uses: ./.github/workflows/report-code-coverage.yml
uses: ./.github/workflows/dt-test-and-report-code-coverage.yml
secrets:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

Expand Down
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -134,4 +134,7 @@ dist
**/.DS_Store


.idea
.idea

# component test report
test_reports/
3 changes: 2 additions & 1 deletion src/cdk/v2/destinations/ortto/procWorkflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ steps:
description: |
Builds common fields in destination payload.
template: |
let phone = .message.().({{{{$.getGenericPaths("phone")}}}});
let commonFields = .message.().({
"fields": {
"str::first": {{{{$.getGenericPaths("firstName")}}}},
Expand All @@ -46,7 +47,7 @@ steps:
"dtz::b": $.getBirthdayObj({{{{$.getGenericPaths("birthday")}}}}),
"str::ei": {{{{$.getGenericPaths("userId")}}}},
"str::language": .context.traits.language || .context.locale,
"phn::phone": {"n": {{{{$.getGenericPaths("phone")}}}}},
"phn::phone": phone ? {"n": phone},
"bol::gdpr": .context.traits.gdpr ?? true,
"bol::p": .context.traits.emailConsent || false,
"bol::sp": .context.traits.smsConsent || false,
Expand Down
49 changes: 30 additions & 19 deletions src/cdk/v2/destinations/the_trade_desk/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const {
getSuccessRespEvents,
removeUndefinedAndNullValues,
handleRtTfSingleEventError,
isEmptyObject,
} = require('../../../../v0/util');
const tradeDeskConfig = require('./config');

Expand Down Expand Up @@ -59,34 +60,44 @@ const processRecordInputs = (inputs, destination) => {
const successMetadata = [];
const errorResponseList = [];

const error = new InstrumentationError('Invalid action type');
const invalidActionTypeError = new InstrumentationError('Invalid action type');
const emptyFieldsError = new InstrumentationError('Fields cannot be empty');

inputs.forEach((input) => {
const { fields, action } = input.message;
const isInsertOrDelete = action === 'insert' || action === 'delete';

if (isInsertOrDelete) {
successMetadata.push(input.metadata);
const data = [
{
Name: Config.audienceId,
TTLInMinutes: action === 'insert' ? ttlInMin(Config.ttlInDays) : 0,
},
];

Object.keys(fields).forEach((id) => {
const value = fields[id];
if (value) {
// adding only non empty ID's
items.push({ [id]: value, Data: data });
}
});
} else {
errorResponseList.push(handleRtTfSingleEventError(input, error, {}));
if (!isInsertOrDelete) {
errorResponseList.push(handleRtTfSingleEventError(input, invalidActionTypeError, {}));
return;
}

if (isEmptyObject(fields)) {
errorResponseList.push(handleRtTfSingleEventError(input, emptyFieldsError, {}));
return;
}

successMetadata.push(input.metadata);
const data = [
{
Name: Config.audienceId,
TTLInMinutes: action === 'insert' ? ttlInMin(Config.ttlInDays) : 0,
},
];

Object.keys(fields).forEach((id) => {
const value = fields[id];
if (value) {
// adding only non empty ID's
items.push({ [id]: value, Data: data });
}
});
});

const payloads = batchResponseBuilder(items, Config);
if (payloads.length === 0) {
return errorResponseList;
}

const response = getSuccessRespEvents(payloads, successMetadata, destination, true);
return [response, ...errorResponseList];
Expand Down
10 changes: 10 additions & 0 deletions src/v0/destinations/tiktok_ads/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ const ConfigCategory = {
type: 'track',
name: 'TikTokTrack',
},
TRACK_V2: {
type: 'track',
name: 'TikTokTrackV2',
},
};

const PARTNER_NAME = 'RudderStack';
Expand All @@ -32,12 +36,18 @@ const eventNameMapping = {

const mappingConfig = getMappingConfig(ConfigCategory, __dirname);

// tiktok docs for max batch size for events 2.0: https://business-api.tiktok.com/portal/docs?id=1771100779668482
const maxBatchSizeV2 = 1000;
const trackEndpointV2 = 'https://business-api.tiktok.com/open_api/v1.3/event/track/';
module.exports = {
TRACK_ENDPOINT,
BATCH_ENDPOINT,
MAX_BATCH_SIZE,
PARTNER_NAME,
trackMapping: mappingConfig[ConfigCategory.TRACK.name],
trackMappingV2: mappingConfig[ConfigCategory.TRACK_V2.name],
eventNameMapping,
DESTINATION: 'TIKTOK_ADS',
trackEndpointV2,
maxBatchSizeV2,
};
16 changes: 16 additions & 0 deletions src/v0/destinations/tiktok_ads/data/TikTokTrack.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,22 @@
"type": "toString"
}
},
{
"destKey": "properties.shop_id",
"sourceKeys": ["properties.shop_id", "properties.shopId"],
"required": false,
"metadata": {
"type": "toString"
}
},
{
"destKey": "properties.order_id",
"sourceKeys": ["properties.order_id", "properties.orderId"],
"required": false,
"metadata": {
"type": "toString"
}
},
{
"destKey": "properties.content_type",
"sourceKeys": ["properties.contentType", "properties.content_type"],
Expand Down
133 changes: 133 additions & 0 deletions src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
[
{
"destKey": "event_id",
"sourceKeys": ["properties.eventId", "properties.event_id", "messageId"],
"required": false
},
{
"destKey": "event_time",
"sourceKeys": "timestamp",
"sourceFromGenericMap": true,
"required": true,
"metadata": {
"type": "secondTimestamp"
}
},
{
"destKey": "limited_data_use",
"sourceKeys": "properties.limited_data_use",
"required": false
},
{
"destKey": "properties.contents",
"sourceKeys": "properties.contents",
"required": false
},
{
"destKey": "properties.content_type",
"sourceKeys": ["properties.contentType", "properties.content_type"],
"metadata": {
"defaultValue": "product"
}
},
{
"destKey": "properties.shop_id",
"sourceKeys": ["properties.shop_id", "properties.shopId"],
"required": false,
"metadata": {
"type": "toString"
}
},
{
"destKey": "properties.order_id",
"sourceKeys": ["properties.order_id", "properties.orderId"],
"required": false,
"metadata": {
"type": "toString"
}
},
{
"destKey": "properties.currency",
"sourceKeys": "properties.currency",
"required": false
},
{
"destKey": "properties.value",
"sourceKeys": "properties.value",
"required": false
},
{
"destKey": "properties.description",
"sourceKeys": "properties.description",
"required": false
},
{
"destKey": "properties.query",
"sourceKeys": "properties.query",
"required": false
},
{
"destKey": "page.url",
"sourceKeys": ["properties.context.page.url", "properties.url", "context.page.url"],
"required": true
},
{
"destKey": "page.referrer",
"sourceKeys": [
"properties.context.page.referrer",
"properties.referrer",
"context.page.referrer"
],
"required": false
},
{
"destKey": "user.locale",
"sourceKeys": ["properties.context.user.locale", "context.locale"],
"required": false
},
{
"destKey": "user.ttclid",
"sourceKeys": "properties.ttclid",
"required": false
},
{
"destKey": "user.ttp",
"sourceKeys": ["properties.context.user.ttp", "properties.ttp"],
"required": false
},
{
"destKey": "user.email",
"sourceKeys": [
"properties.context.user.email",
"context.user.email",
"traits.email",
"context.traits.email",
"properties.email"
],
"metadata": {
"type": ["trim", "toLower"]
},
"required": false
},
{
"destKey": "user.phone",
"sourceKeys": [
"properties.context.user.phone",
"traits.phone",
"context.traits.phone",
"properties.phone"
],
"sourceFromGenericMap": false,
"required": false
},
{
"destKey": "user.ip",
"sourceKeys": ["properties.context.user.ip", "context.ip", "request_ip"],
"required": false
},
{
"destKey": "user.user_agent",
"sourceKeys": ["properties.context.user.userAgent", "context.userAgent"],
"required": false
}
]
Loading

0 comments on commit 7b80f4e

Please sign in to comment.