feat: add v0 of sync service (#547)

fern/definition/common/errors.yml

@@ -16,3 +16,6 @@ errors:
     BadRequestError:
         status-code: 400
         type: BaseError
+    NotImplementedError:
+        status-code: 500
+        type: BaseError

fern/definition/sync.yml

@@ -0,0 +1,39 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+
+imports:
+    errors: common/errors.yml
+    types: common/types.yml
+
+types:
+    TriggerSyncResponse:
+        properties:
+            status: types.ResponseStatus
+
+service:
+    base-path: /sync
+    auth: false
+    audiences:
+        - external
+    endpoints:
+        triggerSync:
+            docs: Trigger sync for a specific tenant
+            method: POST
+            path: ''
+            request:
+                name: TriggerSyncRequest
+                headers:
+                    x-revert-api-token:
+                        type: string
+                        docs: Your official API key for accessing revert apis.
+                    x-revert-t-id:
+                        type: string
+                        docs: The unique customer id used when the customer linked their account.
+                    x-connection-api-key:
+                        type: optional<string>
+                        docs: API key for third party provider
+            response: TriggerSyncResponse
+            errors:
+                - errors.UnAuthorizedError
+                - errors.InternalServerError
+                - errors.NotFoundError
+                - errors.NotImplementedError

packages/backend/.env.example

@@ -40,4 +40,6 @@ JIRA_CLIENT_ID=
 JIRA_CLIENT_SECRET=
 MS_DYNAMICS_SALES_CLIENT_ID=
 MS_DYNAMICS_SALES_CLIENT_SECRET=
-MS_DYNAMICS_SALES_ORG_URL=
+MS_DYNAMICS_SALES_ORG_URL=
+OPEN_INT_API_KEY=
+TWENTY_ACCOUNT_ID=

packages/backend/config.ts

@@ -50,6 +50,8 @@ const config = {
     MS_DYNAMICS_SALES_ORG_URL: process.env.MS_DYNAMICS_SALES_ORG_URL!,
     BITBUCKET_CLIENT_ID: process.env.BITBUCKET_CLIENT_ID!,
     BITBUCKET_CLIENT_SECRET: process.env.BITBUCKET_CLIENT_SECRET!,
+    OPEN_INT_API_KEY: process.env.OPEN_INT_API_KEY,
+    TWENTY_ACCOUNT_ID: process.env.TWENTY_ACCOUNT_ID,
 };
 
 export default config;

packages/backend/oas/openapi.yml

@@ -2775,6 +2775,57 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/commonBaseError'
+  /sync:
+    post:
+      description: Trigger sync for a specific tenant
+      operationId: sync_triggerSync
+      tags:
+        - Sync
+      parameters:
+        - name: x-revert-api-token
+          in: header
+          description: Your official API key for accessing revert apis.
+          required: true
+          schema:
+            type: string
+        - name: x-revert-t-id
+          in: header
+          description: The unique customer id used when the customer linked their account.
+          required: true
+          schema:
+            type: string
+        - name: x-connection-api-key
+          in: header
+          description: API key for third party provider
+          required: false
+          schema:
+            type: string
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/TriggerSyncResponse'
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/commonBaseError'
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/commonBaseError'
+        '500':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/commonBaseError'
   /ticket/collections:
     get:
       description: Get all the collections
@@ -4322,7 +4373,7 @@ components:
           type: string
         tp_customer_id:
           type: string
-          description: The email id of the user who connected the app.
+          description: The emailId or a unique ID id of the user who connected the app.
         t_id:
           type: string
         tp_account_url:
@@ -4334,6 +4385,7 @@ components:
           type: string
         app_id:
           type: string
+          description: Can be obtained from the integration dashboard.
       required:
         - tp_id
         - tp_access_token
@@ -5171,6 +5223,14 @@ components:
       enum:
         - active
         - inactive
+    TriggerSyncResponse:
+      title: TriggerSyncResponse
+      type: object
+      properties:
+        status:
+          $ref: '#/components/schemas/commonResponseStatus'
+      required:
+        - status
     ticketGetCollectionsResponse:
       title: ticketGetCollectionsResponse
       type: object

packages/backend/package.json

@@ -56,6 +56,8 @@
     },
     "dependencies": {
         "@linear/sdk": "^13.0.0",
+        "@opensdks/runtime": "0.0.16",
+        "@opensdks/sdk-venice": "0.0.14",
         "@prisma/client": "^4.16.0",
         "@sentry/node": "^7.55.2",
         "@shortloop/node": "0.0.12",

packages/backend/routes/index.ts

@@ -43,6 +43,7 @@ import { userServiceTicket } from '../services/ticket/user';
 import { collectionServiceTicket } from '../services/ticket/collection';
 import { commentServiceTicket } from '../services/ticket/comment';
 import { proxyServiceTicket } from '../services/ticket/proxy';
+import { syncService } from '../services/sync';
 
 const router = express.Router();
 
@@ -178,6 +179,7 @@ register(router, {
         collection: collectionServiceTicket,
         proxy: proxyServiceTicket,
     },
+    sync: syncService,
 });
 
 export default router;

packages/backend/services/custom/twenty.ts

@@ -0,0 +1,141 @@
+import { Connection } from '../../generated/typescript/api/resources/common/index.js';
+import config from '../../config';
+const { initSDK } = require('@opensdks/runtime');
+const { veniceSdkDef } = require('@opensdks/sdk-venice');
+
+const syncTwentyConnection = async (connection: Connection, connectionAPIKey: string) => {
+    const connectionId = connection.t_id;
+    const openIntApiKey = config.OPEN_INT_API_KEY;
+    if (!config.OPEN_INT_API_KEY || !connectionAPIKey) {
+        console.log('Credentials absent to make this sync happen for: ', connection.t_id, ' returning early');
+        return;
+    }
+    const venice = initSDK(
+        {
+            ...veniceSdkDef,
+            oasMeta: {
+                ...veniceSdkDef.oasMeta,
+                // servers: [{ url: 'http://localhost:4000/api/v0' }],
+            },
+        },
+        {
+            headers: {
+                'x-apikey': openIntApiKey,
+            },
+        }
+    );
+
+    const twentyAccessToken = connectionAPIKey;
+
+    const connectorConfig = await venice.GET('/core/connector_config');
+
+    const getConnectorConfig = (cName: string, connectorConfig: any) =>
+        connectorConfig?.data?.filter((c: any) => c.connectorName === cName)[0];
+
+    const twenty = getConnectorConfig('twenty', connectorConfig);
+    const revert = getConnectorConfig('revert', connectorConfig);
+
+    let revertResource = await venice.GET('/core/resource', {
+        params: {
+            query: {
+                connectorConfigId: revert.id,
+                endUserId: connectionId,
+            },
+        },
+    });
+
+    let twentyResource = await venice.GET('/core/resource', {
+        params: {
+            query: {
+                connectorConfigId: twenty.id,
+                endUserId: connectionId,
+            },
+        },
+    });
+
+    let twentyResourceId = twentyResource?.data[0]?.id;
+    let revertResourceId = revertResource?.data[0]?.id;
+
+    twentyResource.data = twentyResource?.data[0];
+    revertResource.data = revertResource?.data[0];
+
+    // Create a twenty resource for this connection
+    if (!twentyResourceId) {
+        twentyResourceId = await venice.POST('/core/resource', {
+            body: {
+                connectorName: twenty.connectorName,
+                displayName: null,
+                endUserId: connectionId,
+                connectorConfigId: twenty.id,
+                integrationId: null,
+                settings: { access_token: twentyAccessToken },
+            },
+        });
+
+        console.log('Created twenty resourceId', twentyResourceId.data);
+
+        twentyResource = await venice.GET(`/core/resource/${twentyResourceId.data}`);
+
+        console.log('Created twenty resource', twentyResource);
+    }
+
+    //  Create a revert resource for this connection in this environment of Twenty if not created in Twenty’s organisation,
+    if (!revertResourceId) {
+        revertResourceId = await venice.POST('/core/resource', {
+            body: {
+                connectorName: revert.connectorName,
+                displayName: null,
+                endUserId: connectionId,
+                connectorConfigId: revert.id,
+                integrationId: null,
+                settings: { tenant_id: connectionId },
+            },
+        });
+        console.log('Created revert resourceId', revertResourceId.data);
+        revertResource = await venice.GET(`/core/resource/${revertResourceId.data}`);
+
+        console.log('Created revert resource', revertResource);
+    }
+
+    console.log('REVERT RESOURCE', revertResourceId);
+
+    console.log('TWENTY RESOURCE', twentyResourceId);
+
+    // Create a pipeline between them if not created,
+    // Trigger this created pipeline
+    const syncPipeline = await venice.GET('/core/pipeline', {
+        params: {
+            query: {
+                resourceIds: [twentyResource?.data, revertResource?.data],
+            },
+        },
+    });
+
+    if (!syncPipeline?.data[0]) {
+        const createdPipeline = await venice.POST('/core/pipeline', {
+            body: {
+                id: `pipe_${connectionId}`,
+                sourceId: revertResource?.data.id,
+                destinationId: twentyResource?.data.id,
+            },
+        });
+        syncPipeline.data = [createdPipeline.data];
+        console.log('Created pipeline', syncPipeline?.data);
+    }
+
+    console.log('PIPELINE', syncPipeline?.data[0]);
+
+    void venice
+        .POST(`/core/pipeline/${syncPipeline?.data[0]?.id}/_sync`, {
+            params: {
+                async: true,
+            },
+        })
+        .then()
+        .catch((e: Error) => console.log('Error', e))
+        .finally(() => {
+            console.log('TRIGGERED');
+        });
+};
+
+export { syncTwentyConnection };

packages/backend/services/sync.ts

@@ -0,0 +1,28 @@
+import config from '../config';
+import { Connection, NotImplementedError } from '../generated/typescript/api/resources/common';
+import { SyncService } from '../generated/typescript/api/resources/sync/service/SyncService';
+import revertAuthMiddleware from '../helpers/authMiddleware';
+import revertTenantMiddleware from '../helpers/tenantIdMiddleware';
+import { syncTwentyConnection } from './custom/twenty';
+
+const syncService = new SyncService(
+    {
+        async triggerSync(req, res) {
+            const account = res.locals.account;
+            if (account.id === config.TWENTY_ACCOUNT_ID) {
+                const { 'x-connection-api-key': twentyAPIKey } = req.headers;
+                const connection = res.locals.connection as Connection;
+                await syncTwentyConnection(connection, twentyAPIKey as string);
+                res.send({
+                    status: 'ok',
+                });
+            } else
+                throw new NotImplementedError({
+                    error: 'Syncing is not yet available for your account, sorry! Contact us on discord or email to get access!',
+                });
+        },
+    },
+    [revertAuthMiddleware(), revertTenantMiddleware()]
+);
+
+export { syncService };