Connectors batch get (#6078)

* connectors: batch get API

* connectors API

* public-urls batch get

* clean-up:

* lint

connectors/src/api/get_connector.ts

@@ -2,6 +2,7 @@ import type {
   ConnectorType,
   WithConnectorsAPIErrorReponse,
 } from "@dust-tt/types";
+import { isConnectorProvider } from "@dust-tt/types";
 import type { Request, Response } from "express";
 
 import { GithubDiscussion, GithubIssue } from "@connectors/lib/models/github";
@@ -72,3 +73,46 @@ const _getConnector = async (
 };
 
 export const getConnectorAPIHandler = withLogging(_getConnector);
+
+type GetConnectorsResponseBody = WithConnectorsAPIErrorReponse<ConnectorType[]>;
+
+const _getConnectors = async (
+  req: Request<Record<string, string>, GetConnectorsResponseBody, undefined>,
+  res: Response<GetConnectorsResponseBody>
+) => {
+  if (
+    typeof req.query.provider !== "string" ||
+    !isConnectorProvider(req.query.provider)
+  ) {
+    return apiError(req, res, {
+      status_code: 400,
+      api_error: {
+        type: "unknown_connector_provider",
+        message: `Unknown connector provider ${req.params.provider}`,
+      },
+    });
+  }
+
+  if (typeof req.query.connector_id === "string") {
+    req.query.connector_id = [req.query.connector_id];
+  }
+
+  if (!Array.isArray(req.query.connector_id)) {
+    return apiError(req, res, {
+      status_code: 400,
+      api_error: {
+        type: "invalid_request_error",
+        message: `Expecting connector_id to be passed as query parameters`,
+      },
+    });
+  }
+
+  const connectors = await ConnectorResource.fetchByIds(
+    req.query.provider,
+    req.query.connector_id as string[]
+  );
+
+  return res.status(200).json(connectors.map((c) => c.toJSON()));
+};
+
+export const getConnectorsAPIHandler = withLogging(_getConnectors);

connectors/src/api_server.ts

@@ -8,7 +8,10 @@ import { adminAPIHandler } from "@connectors/api/admin";
 import { patchConnectorConfigurationAPIHandler } from "@connectors/api/configuration";
 import { createConnectorAPIHandler } from "@connectors/api/create_connector";
 import { deleteConnectorAPIHandler } from "@connectors/api/delete_connector";
-import { getConnectorAPIHandler } from "@connectors/api/get_connector";
+import {
+  getConnectorAPIHandler,
+  getConnectorsAPIHandler,
+} from "@connectors/api/get_connector";
 import { getConnectorPermissionsAPIHandler } from "@connectors/api/get_connector_permissions";
 import { getContentNodesParentsAPIHandler } from "@connectors/api/get_content_node_parents";
 import { getContentNodesAPIHandler } from "@connectors/api/get_content_nodes";
@@ -95,6 +98,7 @@ export function startServer(port: number) {
   app.post("/connectors/resume/:connector_id", resumeConnectorAPIHandler);
   app.delete("/connectors/delete/:connector_id", deleteConnectorAPIHandler);
   app.get("/connectors/:connector_id", getConnectorAPIHandler);
+  app.get("/connectors", getConnectorsAPIHandler);
   app.post("/connectors/sync/:connector_id", syncConnectorAPIHandler);
   app.get(
     "/connectors/:connector_id/permissions",

connectors/src/resources/connector_resource.ts

@@ -142,11 +142,15 @@ export class ConnectorResource extends BaseResource<ConnectorModel> {
     return c;
   }
 
-  static async fetchByIds(type: ConnectorProvider, ids: ModelId[]) {
+  static async fetchByIds(type: ConnectorProvider, ids: (ModelId | string)[]) {
+    const parsedIds = ids.map((id) =>
+      typeof id === "string" ? parseInt(id, 10) : id
+    );
+
     const blobs = await ConnectorResource.model.findAll({
       where: {
         type,
-        id: ids,
+        id: parsedIds,
       },
     });
 

front/pages/w/[wId]/builder/data-sources/public-urls.tsx

@@ -62,24 +62,36 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{
     providerFilter: "webcrawler",
   });
 
+  const connectorIds = allDataSources
+    .filter(
+      (ds) => ds.connectorProvider === "webcrawler" && ds.connectorId !== null
+    )
+    .map((ds) => ds.connectorId) as string[];
+
   const connectorsAPI = new ConnectorsAPI(logger);
-  const dataSources = await Promise.all(
-    allDataSources
-      .filter((ds) => ds.connectorProvider === "webcrawler")
-      .map(async (ds): Promise<DataSourceWithConnector> => {
-        if (!ds.connectorId) {
-          throw new Error("Connector ID is missing");
-        }
-        const connectorRes = await connectorsAPI.getConnector(ds.connectorId);
-        if (connectorRes.isErr()) {
-          throw new Error("Connector not found");
-        }
-        return {
-          ...ds,
-          connector: connectorRes.value,
-        };
-      })
+
+  const connectorsRes = await connectorsAPI.getConnectors(
+    "webcrawler",
+    connectorIds
   );
+  if (connectorsRes.isErr()) {
+    throw new Error("Failed to fetch connectors");
+  }
+
+  const dataSources = allDataSources
+    .filter((ds) => ds.connectorProvider === "webcrawler")
+    .map((ds) => {
+      const connector = connectorsRes.value.find(
+        (c) => c.id === ds.connectorId
+      );
+      if (!connector) {
+        throw new Error("Connector not found");
+      }
+      return {
+        ...ds,
+        connector,
+      };
+    });
 
   return {
     props: {

types/src/front/lib/connectors_api.ts

@@ -355,6 +355,28 @@ export class ConnectorsAPI {
     return this._resultFromResponse(res);
   }
 
+  async getConnectors(
+    provider: ConnectorProvider,
+    connectorIds: string[]
+  ): Promise<ConnectorsAPIResponse<ConnectorType[]>> {
+    if (connectorIds.length === 0) {
+      return new Ok([]);
+    }
+    const res = await this._fetchWithError(
+      `${CONNECTORS_API}/connectors?provider=${encodeURIComponent(
+        provider
+      )}&${connectorIds
+        .map((id) => `connector_id=${encodeURIComponent(id)}`)
+        .join("&")}`,
+      {
+        method: "GET",
+        headers: this.getDefaultHeaders(),
+      }
+    );
+
+    return this._resultFromResponse(res);
+  }
+
   async setConnectorConfig(
     connectorId: string,
     configKey: string,