front: remove data sources visibility (#3414)

* Remove data sources visibility * fixes * enforce admin only can delete connections in lib/api * Update front/pages/api/w/[wId]/data_sources/[name]/index.ts Co-authored-by: Flavien David <[email protected]> --------- Co-authored-by: Flavien David <[email protected]>
dust-tt · Jan 24, 2024 · 98eeaab · 98eeaab
1 parent 909499e
commit 98eeaab
Show file tree

Hide file tree

Showing 26 changed files with 169 additions and 193 deletions.
diff --git a/front/lib/api/data_sources.ts b/front/lib/api/data_sources.ts
@@ -5,7 +5,6 @@ import type {
   Result,
 } from "@dust-tt/types";
 import { ConnectorsAPI, CoreAPI, Err, Ok } from "@dust-tt/types";
-import { Op } from "sequelize";
 
 import { getMembers } from "@app/lib/api/workspace";
 import type { Authenticator } from "@app/lib/auth";
@@ -19,25 +18,19 @@ export async function getDataSource(
   name: string
 ): Promise<DataSourceType | null> {
   const owner = auth.workspace();
-  if (!owner) {
+
+  // This condition is critical it checks that we can identify the workspace and that the current
+  // auth is a user for this workspace. Checking `auth.isUser()` is critical as it would otherwise
+  // be possible to access data sources without being authenticated.
+  if (!owner || !auth.isUser()) {
     return null;
   }
 
   const dataSource = await DataSource.findOne({
-    where: auth.isUser()
-      ? {
-          workspaceId: owner.id,
-          visibility: {
-            [Op.or]: ["public", "private", "unlisted"],
-          },
-          name,
-        }
-      : {
-          workspaceId: owner.id,
-          // Do not include 'unlisted' here.
-          visibility: "public",
-          name,
-        },
+    where: {
+      workspaceId: owner.id,
+      name,
+    },
   });
 
   if (!dataSource) {
@@ -48,7 +41,6 @@ export async function getDataSource(
     id: dataSource.id,
     name: dataSource.name,
     description: dataSource.description,
-    visibility: dataSource.visibility,
     dustAPIProjectId: dataSource.dustAPIProjectId,
     connectorId: dataSource.connectorId,
     connectorProvider: dataSource.connectorProvider,
@@ -60,23 +52,18 @@ export async function getDataSources(
   auth: Authenticator
 ): Promise<DataSourceType[]> {
   const owner = auth.workspace();
-  if (!owner) {
+
+  // This condition is critical it checks that we can identify the workspace and that the current
+  // auth is a user for this workspace. Checking `auth.isUser()` is critical as it would otherwise
+  // be possible to access data sources without being authenticated.
+  if (!owner || !auth.isUser()) {
     return [];
   }
 
   const dataSources = await DataSource.findAll({
-    where: auth.isUser()
-      ? {
-          workspaceId: owner.id,
-          visibility: {
-            [Op.or]: ["public", "private", "unlisted"],
-          },
-        }
-      : {
-          workspaceId: owner.id,
-          // Do not include 'unlisted' here.
-          visibility: "public",
-        },
+    where: {
+      workspaceId: owner.id,
+    },
     order: [["updatedAt", "DESC"]],
   });
 
@@ -85,35 +72,37 @@ export async function getDataSources(
       id: dataSource.id,
       name: dataSource.name,
       description: dataSource.description,
-      visibility: dataSource.visibility,
       dustAPIProjectId: dataSource.dustAPIProjectId,
       connectorId: dataSource.connectorId,
       connectorProvider: dataSource.connectorProvider,
       assistantDefaultSelected: dataSource.assistantDefaultSelected,
     };
   });
 }
+
 export async function deleteDataSource(
   auth: Authenticator,
   dataSourceName: string
 ): Promise<Result<{ success: true }, APIError>> {
-  const workspace = auth.workspace();
-  if (!workspace) {
+  const owner = auth.workspace();
+  if (!owner) {
     return new Err({
       type: "workspace_not_found",
       message: "Could not find the workspace.",
     });
   }
-  if (!auth.isAdmin()) {
+
+  if (!auth.isBuilder()) {
     return new Err({
       type: "workspace_auth_error",
       message:
-        "Only users that are `admins` for the current workspace can delete data sources.",
+        "Only users that are `builders` for the current workspace can delete data sources.",
     });
   }
+
   const dataSource = await DataSource.findOne({
     where: {
-      workspaceId: workspace.id,
+      workspaceId: owner.id,
       name: dataSourceName,
     },
   });
@@ -126,8 +115,16 @@ export async function deleteDataSource(
 
   const dustAPIProjectId = dataSource.dustAPIProjectId;
 
-  const connectorsAPI = new ConnectorsAPI(logger);
   if (dataSource.connectorId) {
+    if (!auth.isAdmin()) {
+      return new Err({
+        type: "workspace_auth_error",
+        message:
+          "Only users that are `admins` for the current workspace can delete connected data sources.",
+      });
+    }
+
+    const connectorsAPI = new ConnectorsAPI(logger);
     const connDeleteRes = await connectorsAPI.deleteConnector(
       dataSource.connectorId.toString(),
       true
@@ -139,6 +136,7 @@ export async function deleteDataSource(
         return new Err({
           type: "internal_server_error",
           message: `Error deleting connector: ${connDeleteRes.error.error.message}`,
+          connectors_error: connDeleteRes.error,
         });
       }
     }
@@ -153,13 +151,14 @@ export async function deleteDataSource(
     return new Err({
       type: "internal_server_error",
       message: `Error deleting core data source: ${coreDeleteRes.error.message}`,
+      data_source_error: coreDeleteRes.error,
     });
   }
 
   await dataSource.destroy();
 
   await launchScrubDataSourceWorkflow({
-    wId: workspace.sId,
+    wId: owner.sId,
     dustAPIProjectId,
   });
   if (dataSource.connectorProvider)

diff --git a/front/lib/models/data_source.ts b/front/lib/models/data_source.ts
@@ -21,7 +21,6 @@ export class DataSource extends Model<
 
   declare name: string;
   declare description: string | null;
-  declare visibility: "public" | "private";
   declare assistantDefaultSelected: boolean;
   declare dustAPIProjectId: string;
   declare connectorId: string | null;
@@ -55,10 +54,6 @@ DataSource.init(
     description: {
       type: DataTypes.TEXT,
     },
-    visibility: {
-      type: DataTypes.STRING,
-      allowNull: false,
-    },
     assistantDefaultSelected: {
       type: DataTypes.BOOLEAN,
       allowNull: false,
@@ -79,8 +74,7 @@ DataSource.init(
     modelName: "data_source",
     sequelize: front_sequelize,
     indexes: [
-      { fields: ["workspaceId", "visibility"] },
-      { fields: ["workspaceId", "name", "visibility"] },
+      { fields: ["workspaceId", "name"] },
       { fields: ["workspaceId", "name"], unique: true },
     ],
   }

diff --git a/front/migrations/20230522_slack_doc_rename_incident.ts b/front/migrations/20230522_slack_doc_rename_incident.ts
@@ -74,7 +74,6 @@ async function main() {
         let dataSource = await DataSource.create({
           name: dataSourceName,
           description: dataSourceDescription,
-          visibility: "private",
           dustAPIProjectId: dustProject.value.project.project_id.toString(),
           workspaceId: workspaceId,
         });

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/index.ts
@@ -67,7 +67,7 @@ async function handler(
 
   const owner = auth.workspace();
   const plan = auth.plan();
-  if (!owner || !plan) {
+  if (!owner || !plan || !auth.isBuilder()) {
     return apiError(req, res, {
       status_code: 404,
       api_error: {
@@ -115,17 +115,6 @@ async function handler(
       return;
 
     case "POST":
-      if (!auth.isBuilder()) {
-        return apiError(req, res, {
-          status_code: 403,
-          api_error: {
-            type: "data_source_auth_error",
-            message:
-              "You can only alter the data souces of the workspaces for which you are a builder.",
-          },
-        });
-      }
-
       if (dataSource.connectorId && !keyRes.value.isSystem) {
         return apiError(req, res, {
           status_code: 403,

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/parents.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/parents.ts
@@ -25,7 +25,7 @@ async function handler(
   );
 
   const owner = auth.workspace();
-  if (!owner) {
+  if (!owner || !auth.isBuilder()) {
     return apiError(req, res, {
       status_code: 404,
       api_error: {
@@ -49,17 +49,6 @@ async function handler(
 
   switch (req.method) {
     case "POST":
-      if (!auth.isBuilder()) {
-        return apiError(req, res, {
-          status_code: 403,
-          api_error: {
-            type: "data_source_auth_error",
-            message:
-              "You can only alter the data souces of the workspaces for which you are a builder.",
-          },
-        });
-      }
-
       if (
         !req.body ||
         !Array.isArray(req.body.parents) ||

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/index.ts
@@ -26,6 +26,16 @@ async function handler(
     req.query.wId as string
   );
 
+  if (!auth.isBuilder()) {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
+      },
+    });
+  }
+
   const dataSource = await getDataSource(auth, req.query.name as string);
 
   if (!dataSource) {

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/search.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/search.ts
@@ -63,6 +63,16 @@ async function handler(
     req.query.wId as string
   );
 
+  if (!auth.isBuilder()) {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
+      },
+    });
+  }
+
   const dataSource = await getDataSource(auth, req.query.name as string);
 
   if (!dataSource) {

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/index.ts
@@ -28,19 +28,24 @@ async function handler(
 
   const owner = auth.workspace();
   const plan = auth.plan();
-  if (!owner || !plan) {
+  if (!owner || !plan || !auth.isBuilder()) {
     return apiError(req, res, {
       status_code: 404,
       api_error: {
-        type: "workspace_not_found",
-        message: "The workspace you requested was not found.",
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
       },
     });
   }
 
   if (!isActivatedStructuredDB(owner)) {
-    res.status(404).end();
-    return;
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
+      },
+    });
   }
 
   const dataSource = await getDataSource(auth, req.query.name as string);

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/[rId].ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/[rId].ts
@@ -28,12 +28,12 @@ async function handler(
 
   const owner = auth.workspace();
   const plan = auth.plan();
-  if (!owner || !plan) {
+  if (!owner || !plan || !auth.isBuilder()) {
     return apiError(req, res, {
       status_code: 404,
       api_error: {
-        type: "workspace_not_found",
-        message: "The workspace you requested was not found.",
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
       },
     });
   }

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/index.ts
@@ -56,12 +56,12 @@ async function handler(
 
   const owner = auth.workspace();
   const plan = auth.plan();
-  if (!owner || !plan) {
+  if (!owner || !plan || !auth.isBuilder()) {
     return apiError(req, res, {
       status_code: 404,
       api_error: {
-        type: "workspace_not_found",
-        message: "The workspace you requested was not found.",
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
       },
     });
   }

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/index.ts
@@ -42,19 +42,24 @@ async function handler(
 
   const owner = auth.workspace();
   const plan = auth.plan();
-  if (!owner || !plan) {
+  if (!owner || !plan || !auth.isBuilder()) {
     return apiError(req, res, {
       status_code: 404,
       api_error: {
-        type: "workspace_not_found",
-        message: "The workspace you requested was not found.",
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
       },
     });
   }
 
   if (!isActivatedStructuredDB(owner)) {
-    res.status(404).end();
-    return;
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "data_source_not_found",
+        message: "The data source you requested was not found.",
+      },
+    });
   }
 
   const dataSource = await getDataSource(auth, req.query.name as string);