Adding folder nodes mapping the Intercom hierarchy (#9469)

* Handling most folder node creations, need to figure out what to do on revoke and strategy.detroy * rebased on main, now with mimetypes and latest api * Created backfill script * oversights * slightly faster team sync * Moved db calls after core calls, refactored util function * moved upserts to activities * linter * fixed activity import * Fixed backfill collection parents, added dry run option * Teams renamed into Conversation, catching null connector * revert microsoft renaming --------- Co-authored-by: Lucas <[email protected]>
dust-tt · Dec 20, 2024 · 3037491 · 3037491
1 parent 31d6d87
commit 3037491
Show file tree

Hide file tree

Showing 6 changed files with 337 additions and 53 deletions.
diff --git a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts
@@ -0,0 +1,144 @@
+import { makeScript } from "scripts/helpers";
+
+import {
+  getDataSourceNodeMimeType,
+  getHelpCenterCollectionInternalId,
+  getHelpCenterInternalId,
+  getParentIdsForCollection,
+  getTeamInternalId,
+  getTeamsInternalId,
+} from "@connectors/connectors/intercom/lib/utils";
+import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
+import { concurrentExecutor } from "@connectors/lib/async_utils";
+import { upsertDataSourceFolder } from "@connectors/lib/data_sources";
+import {
+  IntercomCollection,
+  IntercomHelpCenter,
+  IntercomTeam,
+  IntercomWorkspace,
+} from "@connectors/lib/models/intercom";
+import { ConnectorResource } from "@connectors/resources/connector_resource";
+
+async function createFolderNodes(execute: boolean) {
+  const connectors = await ConnectorResource.listByType("intercom", {});
+
+  for (const connector of connectors) {
+    const dataSourceConfig = dataSourceConfigFromConnector(connector);
+
+    // Create Teams folder
+    console.log(
+      `[${connector.id}] -> ${JSON.stringify({ folderId: getTeamsInternalId(connector.id), parents: [getTeamsInternalId(connector.id)] })}`
+    );
+    if (execute) {
+      await upsertDataSourceFolder({
+        dataSourceConfig,
+        folderId: getTeamsInternalId(connector.id),
+        parents: [getTeamsInternalId(connector.id)],
+        title: "Conversations",
+        mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"),
+      });
+    }
+
+    const teams = await IntercomTeam.findAll({
+      where: {
+        connectorId: connector.id,
+      },
+    });
+    // Create a team folder for each team
+    await concurrentExecutor(
+      teams,
+      async (team) => {
+        const teamInternalId = getTeamInternalId(connector.id, team.teamId);
+        console.log(
+          `[${connector.id}] -> ${JSON.stringify({ folderId: teamInternalId, parents: [teamInternalId, getTeamsInternalId(connector.id)] })}`
+        );
+        if (execute) {
+          await upsertDataSourceFolder({
+            dataSourceConfig,
+            folderId: teamInternalId,
+            parents: [teamInternalId, getTeamsInternalId(connector.id)],
+            title: team.name,
+            mimeType: getDataSourceNodeMimeType("TEAM"),
+          });
+        }
+      },
+      { concurrency: 16 }
+    );
+
+    // Length = 1, for loop just in case
+    const workspaces = await IntercomWorkspace.findAll({
+      where: {
+        connectorId: connector.id,
+      },
+    });
+
+    for (const workspace of workspaces) {
+      // Length mostly 1
+      const helpCenters = await IntercomHelpCenter.findAll({
+        where: {
+          connectorId: connector.id,
+          intercomWorkspaceId: workspace.intercomWorkspaceId,
+        },
+      });
+
+      for (const helpCenter of helpCenters) {
+        // Create Help Center folder
+        const helpCenterInternalId = getHelpCenterInternalId(
+          connector.id,
+          helpCenter.helpCenterId
+        );
+        console.log(
+          `[${connector.id}] -> ${JSON.stringify({ folderId: helpCenterInternalId, parents: [helpCenterInternalId] })}`
+        );
+        if (execute) {
+          await upsertDataSourceFolder({
+            dataSourceConfig,
+            folderId: helpCenterInternalId,
+            parents: [helpCenterInternalId],
+            title: helpCenter.name,
+            mimeType: getDataSourceNodeMimeType("HELP_CENTER"),
+          });
+        }
+
+        const collections = await IntercomCollection.findAll({
+          where: {
+            connectorId: connector.id,
+            helpCenterId: helpCenter.helpCenterId,
+          },
+        });
+
+        // Create a collection folder for each collection
+        await concurrentExecutor(
+          collections,
+          async (collection) => {
+            const collectionInternalId = getHelpCenterCollectionInternalId(
+              connector.id,
+              collection.collectionId
+            );
+            const collectionParents = await getParentIdsForCollection({
+              connectorId: connector.id,
+              collectionId: collection.collectionId,
+              helpCenterId: helpCenter.helpCenterId,
+            });
+            console.log(
+              `[${connector.id}] -> ${JSON.stringify({ folderId: collectionInternalId, parents: collectionParents })}`
+            );
+            if (execute) {
+              await upsertDataSourceFolder({
+                dataSourceConfig,
+                folderId: collectionInternalId,
+                parents: collectionParents,
+                title: collection.name,
+                mimeType: getDataSourceNodeMimeType("COLLECTION"),
+              });
+            }
+          },
+          { concurrency: 16 }
+        );
+      }
+    }
+  }
+}
+makeScript({}, async ({ execute }) => {
+  await createFolderNodes(execute);
+});
diff --git a/connectors/src/connectors/intercom/lib/utils.ts b/connectors/src/connectors/intercom/lib/utils.ts
@@ -4,6 +4,29 @@ import type {
   IntercomArticleType,
   IntercomCollectionType,
 } from "@connectors/connectors/intercom/lib/types";
+import { IntercomCollection } from "@connectors/lib/models/intercom";
+
+/**
+ * Mimetypes
+ */
+export function getDataSourceNodeMimeType(
+  intercomNodeType:
+    | "COLLECTION"
+    | "TEAM"
+    | "CONVERSATIONS_FOLDER"
+    | "HELP_CENTER"
+): string {
+  switch (intercomNodeType) {
+    case "COLLECTION":
+      return "application/vnd.dust.intercom.collection";
+    case "CONVERSATIONS_FOLDER":
+      return "application/vnd.dust.intercom.teams-folder";
+    case "TEAM":
+      return "application/vnd.dust.intercom.team";
+    case "HELP_CENTER":
+      return "application/vnd.dust.intercom.help-center";
+  }
+}
 
 /**
  * From id to internalId
@@ -113,3 +136,68 @@ export function getConversationInAppUrl(
   const domain = getIntercomDomain(region);
   return `${domain}/a/inbox/${workspaceId}/inbox/conversation/${conversationId}`;
 }
+
+// Parents in the Core datasource should map the internal ids that we use in the permission modal
+// Order is important: We want the id of the article, then all parents collection in order, then the help center
+export async function getParentIdsForArticle({
+  documentId,
+  connectorId,
+  parentCollectionId,
+  helpCenterId,
+}: {
+  documentId: string;
+  connectorId: number;
+  parentCollectionId: string;
+  helpCenterId: string;
+}) {
+  // Get collection parents
+  const collectionParents = await getParentIdsForCollection({
+    connectorId,
+    collectionId: parentCollectionId,
+    helpCenterId,
+  });
+
+  return [documentId, ...collectionParents];
+}
+
+export async function getParentIdsForCollection({
+  connectorId,
+  collectionId,
+  helpCenterId,
+}: {
+  connectorId: number;
+  collectionId: string;
+  helpCenterId: string;
+}) {
+  // Initialize the internal IDs array with the collection ID.
+  const parentIds = [
+    getHelpCenterCollectionInternalId(connectorId, collectionId),
+  ];
+
+  // Fetch and add any parent collection Ids.
+  let currentParentId = collectionId;
+
+  // There's max 2-levels on Intercom.
+  for (let i = 0; i < 2; i++) {
+    const currentParent = await IntercomCollection.findOne({
+      where: {
+        connectorId,
+        collectionId: currentParentId,
+      },
+    });
+
+    if (!currentParent || !currentParent.parentId) {
+      break;
+    }
+
+    currentParentId = currentParent.parentId;
+    parentIds.push(
+      getHelpCenterCollectionInternalId(connectorId, currentParentId)
+    );
+  }
+
+  // Add the help center internal ID.
+  parentIds.push(getHelpCenterInternalId(connectorId, helpCenterId));
+
+  return parentIds;
+}
diff --git a/connectors/src/connectors/intercom/temporal/activities.ts b/connectors/src/connectors/intercom/temporal/activities.ts
@@ -10,6 +10,12 @@ import {
   fetchIntercomTeam,
 } from "@connectors/connectors/intercom/lib/intercom_api";
 import type { IntercomSyncAllConversationsStatus } from "@connectors/connectors/intercom/lib/types";
+import {
+  getDataSourceNodeMimeType,
+  getHelpCenterInternalId,
+  getTeamInternalId,
+  getTeamsInternalId,
+} from "@connectors/connectors/intercom/lib/utils";
 import {
   deleteConversation,
   deleteTeamAndConversations,
@@ -23,6 +29,7 @@ import {
 } from "@connectors/connectors/intercom/temporal/sync_help_center";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { concurrentExecutor } from "@connectors/lib/async_utils";
+import { upsertDataSourceFolder } from "@connectors/lib/data_sources";
 import {
   IntercomConversation,
   IntercomWorkspace,
@@ -160,6 +167,19 @@ export async function syncHelpCenterOnlyActivity({
     return false;
   }
 
+  // Create datasource folder node
+  const helpCenterInternalId = getHelpCenterInternalId(
+    connectorId,
+    helpCenterOnIntercom.id
+  );
+  await upsertDataSourceFolder({
+    dataSourceConfig,
+    folderId: helpCenterInternalId,
+    title: helpCenterOnIntercom.display_name || "Help Center",
+    parents: [helpCenterInternalId],
+    mimeType: getDataSourceNodeMimeType("HELP_CENTER"),
+  });
+
   // If all children collections are not allowed anymore we delete the Help Center data
   const collectionsWithReadPermission = await IntercomCollection.findAll({
     where: {
@@ -481,6 +501,17 @@ export async function syncTeamOnlyActivity({
     name: teamOnIntercom.name,
     lastUpsertedTs: new Date(currentSyncMs),
   });
+
+  // Also make sure a datasource folder node is created for the team
+  const teamInternalId = getTeamInternalId(connectorId, teamOnDB.teamId);
+  await upsertDataSourceFolder({
+    dataSourceConfig: dataSourceConfigFromConnector(connector),
+    folderId: teamInternalId,
+    title: teamOnIntercom.name,
+    parents: [teamInternalId, getTeamsInternalId(connectorId)],
+    mimeType: getDataSourceNodeMimeType("TEAM"),
+  });
+
   return true;
 }
 
@@ -698,3 +729,20 @@ export async function getSyncAllConversationsStatusActivity({
 
   return intercomWorkspace.syncAllConversations;
 }
+
+export async function upsertIntercomTeamsFolderActivity({
+  connectorId,
+}: {
+  connectorId: ModelId;
+}) {
+  const connector = await _getIntercomConnectorOrRaise(connectorId);
+  const dataSourceConfig = dataSourceConfigFromConnector(connector);
+
+  await upsertDataSourceFolder({
+    dataSourceConfig,
+    folderId: getTeamsInternalId(connectorId),
+    title: "Conversations",
+    parents: [getTeamsInternalId(connectorId)],
+    mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"),
+  });
+}
diff --git a/connectors/src/connectors/intercom/temporal/sync_conversation.ts b/connectors/src/connectors/intercom/temporal/sync_conversation.ts
@@ -17,6 +17,7 @@ import {
 import { concurrentExecutor } from "@connectors/lib/async_utils";
 import {
   deleteDataSourceDocument,
+  deleteDataSourceFolder,
   renderDocumentTitleAndContent,
   renderMarkdownSection,
   upsertDataSourceDocument,
@@ -58,6 +59,12 @@ export async function deleteTeamAndConversations({
     { concurrency: 10 }
   );
 
+  // Delete datasource team node
+  await deleteDataSourceFolder({
+    dataSourceConfig,
+    folderId: getTeamInternalId(connectorId, team.teamId),
+  });
+
   await team.destroy();
 }