Merge branch 'main' into add-provider-visibility-to-node

# Conflicts:
#	core/bin/core_api.rs
#	core/src/data_sources/folder.rs
#	core/src/data_sources/node.rs
#	core/src/stores/postgres.rs
#	core/src/stores/store.rs

connectors/migrations/20241219_backfill_intercom_data_source_folders.ts

@@ -1,4 +1,4 @@
-import { INTERCOM_MIME_TYPES } from "@dust-tt/types";
+import { MIME_TYPES } from "@dust-tt/types";
 import { makeScript } from "scripts/helpers";
 
 import {
@@ -36,7 +36,7 @@ async function createFolderNodes(execute: boolean) {
         parents: [getTeamsInternalId(connector.id)],
         parentId: null,
         title: "Conversations",
-        mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS,
+        mimeType: MIME_TYPES.INTERCOM.TEAMS_FOLDER,
       });
     }
 
@@ -60,7 +60,7 @@ async function createFolderNodes(execute: boolean) {
             parents: [teamInternalId, getTeamsInternalId(connector.id)],
             parentId: getTeamsInternalId(connector.id),
             title: team.name,
-            mimeType: INTERCOM_MIME_TYPES.TEAM,
+            mimeType: MIME_TYPES.INTERCOM.TEAM,
           });
         }
       },
@@ -99,7 +99,7 @@ async function createFolderNodes(execute: boolean) {
             parents: [helpCenterInternalId],
             parentId: null,
             title: helpCenter.name,
-            mimeType: INTERCOM_MIME_TYPES.HELP_CENTER,
+            mimeType: MIME_TYPES.INTERCOM.HELP_CENTER,
           });
         }
 
@@ -133,7 +133,7 @@ async function createFolderNodes(execute: boolean) {
                 parents: collectionParents,
                 parentId: collectionParents[1] || null,
                 title: collection.name,
-                mimeType: INTERCOM_MIME_TYPES.COLLECTION,
+                mimeType: MIME_TYPES.INTERCOM.COLLECTION,
               });
             }
           },

connectors/migrations/20250110_investigate_zendesk_hc.ts

@@ -0,0 +1,100 @@
+import { makeScript } from "scripts/helpers";
+
+import { isZendeskNotFoundError } from "@connectors/connectors/zendesk/lib/errors";
+import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token";
+import {
+  fetchZendeskBrand,
+  fetchZendeskCategoriesInBrand,
+  fetchZendeskCurrentUser,
+  getZendeskBrandSubdomain,
+} from "@connectors/connectors/zendesk/lib/zendesk_api";
+import { ZENDESK_BATCH_SIZE } from "@connectors/connectors/zendesk/temporal/config";
+import { ConnectorResource } from "@connectors/resources/connector_resource";
+import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources";
+
+makeScript({}, async ({ execute }, logger) => {
+  const connectors = await ConnectorResource.listByType("zendesk", {});
+
+  for (const connector of connectors) {
+    const connectorId = connector.id;
+    if (connector.isPaused()) {
+      continue;
+    }
+    const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken(
+      connector.connectionId
+    );
+    const user = await fetchZendeskCurrentUser({ accessToken, subdomain });
+    const brandsOnDb = await ZendeskBrandResource.fetchByConnector(connector);
+    for (const brandOnDb of brandsOnDb) {
+      const { brandId } = brandOnDb;
+      if (!execute) {
+        logger.info({ brandId }, "DRY: Fetching brand");
+        continue;
+      }
+
+      const fetchedBrand = await fetchZendeskBrand({
+        brandId,
+        accessToken,
+        subdomain,
+      });
+      const brandSubdomain = await getZendeskBrandSubdomain({
+        brandId,
+        connectorId,
+        accessToken,
+        subdomain,
+      });
+
+      let couldFetchCategories;
+      try {
+        await fetchZendeskCategoriesInBrand(accessToken, {
+          brandSubdomain,
+          pageSize: ZENDESK_BATCH_SIZE,
+        });
+        couldFetchCategories = true;
+      } catch (e) {
+        if (isZendeskNotFoundError(e)) {
+          couldFetchCategories = false;
+          // if (fetchedBrand?.has_help_center) {
+          //   const url = `https://${brandSubdomain}.zendesk.com/api/v2/help_center/articles.json`;
+          //   const res = await fetch(url, {
+          //     method: "GET",
+          //     headers: {
+          //       Authorization: `Bearer ${accessToken}`,
+          //       "Content-Type": "application/json",
+          //     },
+          //   });
+          //   const text = await res.text();
+          //   logger.error(
+          //     {
+          //       res,
+          //       brandSubdomain,
+          //       status: res.status,
+          //       statusText: res.statusText,
+          //       headers: res.headers,
+          //       text,
+          //       body: res.body,
+          //     },
+          //     "Failed to fetch categories"
+          //   );
+          //   await res.json();
+          // }
+        } else {
+          throw e;
+        }
+      }
+      logger.info(
+        {
+          connectorId,
+          brandId,
+          couldFetchCategories,
+          brandSubdomain,
+          hasHelpCenter: fetchedBrand?.has_help_center,
+          helpCenterState: fetchedBrand?.help_center_state,
+          userRole: user.role,
+          userActive: user.active,
+        },
+        `FETCH`
+      );
+    }
+  }
+});

connectors/src/api_server.ts

@@ -13,7 +13,6 @@ import {
   getConnectorsAPIHandler,
 } from "@connectors/api/get_connector";
 import { getConnectorPermissionsAPIHandler } from "@connectors/api/get_connector_permissions";
-import { getContentNodesParentsAPIHandler } from "@connectors/api/get_content_node_parents";
 import { getContentNodesAPIHandler } from "@connectors/api/get_content_nodes";
 import { pauseConnectorAPIHandler } from "@connectors/api/pause_connector";
 import { resumeConnectorAPIHandler } from "@connectors/api/resume_connector";
@@ -112,11 +111,6 @@ export function startServer(port: number) {
     "/connectors/:connector_id/permissions",
     getConnectorPermissionsAPIHandler
   );
-  app.post(
-    // must be POST because of body
-    "/connectors/:connector_id/content_nodes/parents",
-    getContentNodesParentsAPIHandler
-  );
   app.post(
     // must be POST because of body
     "/connectors/:connector_id/content_nodes",

connectors/src/connectors/confluence/lib/permissions.ts

@@ -46,15 +46,13 @@ export function createContentNodeFromSpace(
   const spaceId = isConfluenceSpaceModel(space) ? space.spaceId : space.id;
 
   return {
-    provider: "confluence",
     internalId: makeSpaceInternalId(spaceId),
     parentInternalId: null,
     type: "folder",
     title: space.name || "Unnamed Space",
     sourceUrl: `${baseUrl}/wiki${urlSuffix}`,
     expandable: isExpandable,
     permission,
-    dustDocumentId: null,
     lastUpdatedAt: null,
   };
 }
@@ -66,7 +64,6 @@ export function createContentNodeFromPage(
   isExpandable = false
 ): ContentNode {
   return {
-    provider: "confluence",
     internalId: makePageInternalId(page.pageId),
     parentInternalId:
       parent.type === "space"
@@ -77,7 +74,6 @@ export function createContentNodeFromPage(
     sourceUrl: `${baseUrl}/wiki${page.externalUrl}`,
     expandable: isExpandable,
     permission: "read",
-    dustDocumentId: null,
     lastUpdatedAt: null,
   };
 }

connectors/src/connectors/confluence/temporal/activities.ts

@@ -1,8 +1,8 @@
 import type { ModelId } from "@dust-tt/types";
 import {
-  CONFLUENCE_MIME_TYPES,
   ConfluenceClientError,
   isConfluenceNotFoundError,
+  MIME_TYPES,
 } from "@dust-tt/types";
 import { Op } from "sequelize";
 import TurndownService from "turndown";
@@ -222,7 +222,7 @@ export async function confluenceUpsertSpaceFolderActivity({
     parents: [makeSpaceInternalId(spaceId)],
     parentId: null,
     title: spaceName,
-    mimeType: CONFLUENCE_MIME_TYPES.SPACE,
+    mimeType: MIME_TYPES.CONFLUENCE.SPACE,
   });
 }
 
@@ -329,7 +329,7 @@ async function upsertConfluencePageToDataSource({
       timestampMs: lastPageVersionCreatedAt.getTime(),
       upsertContext: { sync_type: syncType },
       title: page.title,
-      mimeType: CONFLUENCE_MIME_TYPES.PAGE,
+      mimeType: MIME_TYPES.CONFLUENCE.PAGE,
       async: true,
     });
   }