diff --git a/connectors/src/api/get_connector.ts b/connectors/src/api/get_connector.ts index e92ddd115898..a1e4e1a66468 100644 --- a/connectors/src/api/get_connector.ts +++ b/connectors/src/api/get_connector.ts @@ -2,6 +2,7 @@ import type { ConnectorType, WithConnectorsAPIErrorReponse, } from "@dust-tt/types"; +import { isConnectorProvider } from "@dust-tt/types"; import type { Request, Response } from "express"; import { GithubDiscussion, GithubIssue } from "@connectors/lib/models/github"; @@ -72,3 +73,46 @@ const _getConnector = async ( }; export const getConnectorAPIHandler = withLogging(_getConnector); + +type GetConnectorsResponseBody = WithConnectorsAPIErrorReponse; + +const _getConnectors = async ( + req: Request, GetConnectorsResponseBody, undefined>, + res: Response +) => { + if ( + typeof req.query.provider !== "string" || + !isConnectorProvider(req.query.provider) + ) { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "unknown_connector_provider", + message: `Unknown connector provider ${req.params.provider}`, + }, + }); + } + + if (typeof req.query.connector_id === "string") { + req.query.connector_id = [req.query.connector_id]; + } + + if (!Array.isArray(req.query.connector_id)) { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: `Expecting connector_id to be passed as query parameters`, + }, + }); + } + + const connectors = await ConnectorResource.fetchByIds( + req.query.provider, + req.query.connector_id as string[] + ); + + return res.status(200).json(connectors.map((c) => c.toJSON())); +}; + +export const getConnectorsAPIHandler = withLogging(_getConnectors); diff --git a/connectors/src/api_server.ts b/connectors/src/api_server.ts index 75c25c090ef7..bd9fa77e3d65 100644 --- a/connectors/src/api_server.ts +++ b/connectors/src/api_server.ts @@ -8,7 +8,10 @@ import { adminAPIHandler } from "@connectors/api/admin"; import { patchConnectorConfigurationAPIHandler } from "@connectors/api/configuration"; import { createConnectorAPIHandler } from "@connectors/api/create_connector"; import { deleteConnectorAPIHandler } from "@connectors/api/delete_connector"; -import { getConnectorAPIHandler } from "@connectors/api/get_connector"; +import { + getConnectorAPIHandler, + getConnectorsAPIHandler, +} from "@connectors/api/get_connector"; import { getConnectorPermissionsAPIHandler } from "@connectors/api/get_connector_permissions"; import { getContentNodesParentsAPIHandler } from "@connectors/api/get_content_node_parents"; import { getContentNodesAPIHandler } from "@connectors/api/get_content_nodes"; @@ -95,6 +98,7 @@ export function startServer(port: number) { app.post("/connectors/resume/:connector_id", resumeConnectorAPIHandler); app.delete("/connectors/delete/:connector_id", deleteConnectorAPIHandler); app.get("/connectors/:connector_id", getConnectorAPIHandler); + app.get("/connectors", getConnectorsAPIHandler); app.post("/connectors/sync/:connector_id", syncConnectorAPIHandler); app.get( "/connectors/:connector_id/permissions", diff --git a/connectors/src/resources/connector_resource.ts b/connectors/src/resources/connector_resource.ts index 8efecf865fe0..2e1c50f034a9 100644 --- a/connectors/src/resources/connector_resource.ts +++ b/connectors/src/resources/connector_resource.ts @@ -142,11 +142,15 @@ export class ConnectorResource extends BaseResource { return c; } - static async fetchByIds(type: ConnectorProvider, ids: ModelId[]) { + static async fetchByIds(type: ConnectorProvider, ids: (ModelId | string)[]) { + const parsedIds = ids.map((id) => + typeof id === "string" ? parseInt(id, 10) : id + ); + const blobs = await ConnectorResource.model.findAll({ where: { type, - id: ids, + id: parsedIds, }, }); diff --git a/front/pages/w/[wId]/builder/data-sources/public-urls.tsx b/front/pages/w/[wId]/builder/data-sources/public-urls.tsx index a9061ea4432a..857259052dcb 100644 --- a/front/pages/w/[wId]/builder/data-sources/public-urls.tsx +++ b/front/pages/w/[wId]/builder/data-sources/public-urls.tsx @@ -62,24 +62,36 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{ providerFilter: "webcrawler", }); + const connectorIds = allDataSources + .filter( + (ds) => ds.connectorProvider === "webcrawler" && ds.connectorId !== null + ) + .map((ds) => ds.connectorId) as string[]; + const connectorsAPI = new ConnectorsAPI(logger); - const dataSources = await Promise.all( - allDataSources - .filter((ds) => ds.connectorProvider === "webcrawler") - .map(async (ds): Promise => { - if (!ds.connectorId) { - throw new Error("Connector ID is missing"); - } - const connectorRes = await connectorsAPI.getConnector(ds.connectorId); - if (connectorRes.isErr()) { - throw new Error("Connector not found"); - } - return { - ...ds, - connector: connectorRes.value, - }; - }) + + const connectorsRes = await connectorsAPI.getConnectors( + "webcrawler", + connectorIds ); + if (connectorsRes.isErr()) { + throw new Error("Failed to fetch connectors"); + } + + const dataSources = allDataSources + .filter((ds) => ds.connectorProvider === "webcrawler") + .map((ds) => { + const connector = connectorsRes.value.find( + (c) => c.id === ds.connectorId + ); + if (!connector) { + throw new Error("Connector not found"); + } + return { + ...ds, + connector, + }; + }); return { props: { diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts index bfe596c17704..95be2abce956 100644 --- a/types/src/front/lib/connectors_api.ts +++ b/types/src/front/lib/connectors_api.ts @@ -355,6 +355,28 @@ export class ConnectorsAPI { return this._resultFromResponse(res); } + async getConnectors( + provider: ConnectorProvider, + connectorIds: string[] + ): Promise> { + if (connectorIds.length === 0) { + return new Ok([]); + } + const res = await this._fetchWithError( + `${CONNECTORS_API}/connectors?provider=${encodeURIComponent( + provider + )}&${connectorIds + .map((id) => `connector_id=${encodeURIComponent(id)}`) + .join("&")}`, + { + method: "GET", + headers: this.getDefaultHeaders(), + } + ); + + return this._resultFromResponse(res); + } + async setConnectorConfig( connectorId: string, configKey: string,