[connectors] - refacto: CSV enabled toggle (#6567)

* [connectors] - feature: support CSV file sync for Microsoft connector - Replace connector parameter with csvEnabled to directly accept CSV sync configuration - Remove dependency of isCsvEnabled function by including csvEnabled in the configuration model [front] - feature: display CSV file sync configuration in UI - Add front-end support for toggling CSV file sync for both Google Drive and Microsoft data sources - Retrieve CSV sync configuration state from the back-end to reflect the current settings in the UI * [connectors] - feature: add CSV file support in Google Drive configuration - Introduce a new configuration option to enable processing of CSV files within the Google Drive connector - Default the csvEnabled flag to false, keeping CSV processing disabled by default * [connectors] - feature: add csvEnabled column to configuration tables - Added 'csvEnabled' column with a default value of false to Microsoft and Google Drive configuration tables to support CSV functionality. * [connectors] - feature: add support for CSV syncing in Google Drive and Microsoft connectors - Renamed migration file for consistency in numbering - Implemented CSV enable/disable toggles and handling in Google Drive connector - Added CSV sync configuration logic and added to workflows in Microsoft connector - Removed the old feature flags for CSV synchronization from the codebase [front] - feature: introduce user-facing toggles for CSV file synchronization - Added CSV sync toggles on data source configuration pages for Google Drive and Microsoft connectors [types] - refactor: remove CSV sync related feature flags - Deleted the old CSV sync feature flags 'microsoft_csv_sync' and 'google_csv_sync' from WhitelistableFeature type * [connectors] - feature: add csvEnabled configuration to Google Drive - Enabled the configuration for csv files support in Google Drive integration - Ensured the database migration reflects the addition of the csvEnabled field in the corresponding table --------- Co-authored-by: Jules <[email protected]>
dust-tt · Jul 29, 2024 · e529050 · e529050
1 parent 0bc241b
commit e529050
Show file tree

Hide file tree

Showing 14 changed files with 101 additions and 32 deletions.
diff --git a/connectors/migrations/20231109_2_create_gdrive_config.ts b/connectors/migrations/20231109_2_create_gdrive_config.ts
@@ -12,6 +12,7 @@ async function main() {
     const config = await GoogleDriveConfig.create({
       connectorId: connector.id,
       pdfEnabled: false,
+      csvEnabled: false,
       largeFilesEnabled: false,
     });
     console.log(

diff --git a/connectors/migrations/db/migration_09.sql b/connectors/migrations/db/migration_09.sql
@@ -0,0 +1,3 @@
+-- Migration created on Jul 29, 2024
+ALTER TABLE "public"."microsoft_configurations" ADD COLUMN "csvEnabled" BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE "public"."google_drive_configs" ADD COLUMN "csvEnabled" BOOLEAN NOT NULL DEFAULT false;
diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts
@@ -117,6 +117,7 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager<null> {
     const googleDriveConfigurationBlob = {
       pdfEnabled: false,
       largeFilesEnabled: false,
+      csvEnabled: false,
     };
 
     const connector = await ConnectorResource.makeNew(
@@ -769,7 +770,19 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager<null> {
         }
         return new Ok(void 0);
       }
-
+      case "csvEnabled": {
+        await config.update({
+          csvEnabled: configValue === "true",
+        });
+        const workflowRes = await launchGoogleDriveFullSyncWorkflow(
+          this.connectorId,
+          null
+        );
+        if (workflowRes.isErr()) {
+          return workflowRes;
+        }
+        return new Ok(void 0);
+      }
       case "largeFilesEnabled": {
         await config.update({
           largeFilesEnabled: configValue === "true",
@@ -818,6 +831,9 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager<null> {
       case "largeFilesEnabled": {
         return new Ok(config.largeFilesEnabled ? "true" : "false");
       }
+      case "csvEnabled": {
+        return new Ok(config.csvEnabled ? "true" : "false");
+      }
       default:
         return new Err(new Error(`Invalid config key ${configKey}`));
     }

diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts
@@ -166,7 +166,7 @@ export async function syncFiles(
 
   const mimeTypesToSync = await getMimeTypesToSync({
     pdfEnabled: config?.pdfEnabled || false,
-    connector,
+    csvEnabled: config?.csvEnabled || false,
   });
   const authCredentials = await getAuthObject(connector.connectionId);
   const driveFolder = await getGoogleDriveObject(
@@ -351,7 +351,7 @@ export async function incrementalSync(
     });
     const mimeTypesToSync = await getMimeTypesToSync({
       pdfEnabled: config?.pdfEnabled || false,
-      connector,
+      csvEnabled: config?.csvEnabled || false,
     });
 
     const selectedFoldersIds = await getFoldersToSync(connectorId);

diff --git a/connectors/src/connectors/google_drive/temporal/file.ts b/connectors/src/connectors/google_drive/temporal/file.ts
@@ -182,7 +182,7 @@ export async function syncOneFile(
 
       const mimeTypesToDownload = await getMimeTypesToDownload({
         pdfEnabled: config?.pdfEnabled || false,
-        connector,
+        csvEnabled: config?.csvEnabled || false,
       });
 
       const documentId = getDocumentId(file.id);

diff --git a/connectors/src/connectors/google_drive/temporal/mime_types.ts b/connectors/src/connectors/google_drive/temporal/mime_types.ts
@@ -1,6 +1,4 @@
 import type { GoogleDriveFiles } from "@connectors/lib/models/google_drive";
-import { getEnabledFeatureFlagsMemoized } from "@connectors/lib/workspace";
-import type { ConnectorResource } from "@connectors/resources/connector_resource";
 
 export const MIME_TYPES_TO_EXPORT: { [key: string]: string } = {
   "application/vnd.google-apps.document": "text/plain",
@@ -9,10 +7,10 @@ export const MIME_TYPES_TO_EXPORT: { [key: string]: string } = {
 
 export async function getMimeTypesToDownload({
   pdfEnabled,
-  connector,
+  csvEnabled,
 }: {
   pdfEnabled: boolean;
-  connector: ConnectorResource;
+  csvEnabled: boolean;
 }) {
   const mimeTypes = [
     "text/plain",
@@ -24,7 +22,6 @@ export async function getMimeTypesToDownload({
   if (pdfEnabled) {
     mimeTypes.push("application/pdf");
   }
-  const csvEnabled = await isCsvEnabled(connector);
   if (csvEnabled) {
     mimeTypes.push("text/csv");
   }
@@ -34,14 +31,14 @@ export async function getMimeTypesToDownload({
 
 export async function getMimeTypesToSync({
   pdfEnabled,
-  connector,
+  csvEnabled,
 }: {
   pdfEnabled: boolean;
-  connector: ConnectorResource;
+  csvEnabled: boolean;
 }) {
   const mimeTypes = await getMimeTypesToDownload({
     pdfEnabled,
-    connector,
+    csvEnabled,
   });
   mimeTypes.push(...Object.keys(MIME_TYPES_TO_EXPORT));
   mimeTypes.push("application/vnd.google-apps.folder");
@@ -57,8 +54,3 @@ export function isGoogleDriveFolder(file: GoogleDriveFiles) {
 export function isGoogleDriveSpreadSheetFile(file: { mimeType: string }) {
   return file.mimeType === "application/vnd.google-apps.spreadsheet";
 }
-
-async function isCsvEnabled(connector: ConnectorResource): Promise<boolean> {
-  const enabledFeatureFlags = await getEnabledFeatureFlagsMemoized(connector);
-  return !!enabledFeatureFlags.includes("google_csv_sync");
-}
diff --git a/connectors/src/connectors/microsoft/index.ts b/connectors/src/connectors/microsoft/index.ts
@@ -78,6 +78,7 @@ export class MicrosoftConnectorManager extends BaseConnectorManager<null> {
 
     const microsoftConfigurationBlob = {
       pdfEnabled: false,
+      csvEnabled: false,
       largeFilesEnabled: false,
     };
 
@@ -532,6 +533,18 @@ export class MicrosoftConnectorManager extends BaseConnectorManager<null> {
         }
         return new Ok(undefined);
       }
+      case "csvEnabled": {
+        await config.update({
+          csvEnabled: configValue === "true",
+        });
+        const workflowRes = await launchMicrosoftFullSyncWorkflow(
+          this.connectorId
+        );
+        if (workflowRes.isErr()) {
+          return workflowRes;
+        }
+        return new Ok(undefined);
+      }
 
       case "largeFilesEnabled": {
         await config.update({
@@ -578,6 +591,9 @@ export class MicrosoftConnectorManager extends BaseConnectorManager<null> {
       case "pdfEnabled": {
         return new Ok(config.pdfEnabled ? "true" : "false");
       }
+      case "csvEnabled": {
+        return new Ok(config.csvEnabled ? "true" : "false");
+      }
       case "largeFilesEnabled": {
         return new Ok(config.largeFilesEnabled ? "true" : "false");
       }

diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts
@@ -334,7 +334,7 @@ export async function syncFiles({
 
   const mimeTypesToSync = await getMimeTypesToSync({
     pdfEnabled: providerConfig.pdfEnabled || false,
-    connector,
+    csvEnabled: providerConfig.csvEnabled || false,
   });
   const filesToSync = children.filter(
     (item) =>

diff --git a/connectors/src/connectors/microsoft/temporal/file.ts b/connectors/src/connectors/microsoft/temporal/file.ts
@@ -125,7 +125,7 @@ export async function syncOneFile({
 
   const mimeTypesToSync = await getMimeTypesToSync({
     pdfEnabled: providerConfig.pdfEnabled || false,
-    connector,
+    csvEnabled: providerConfig.csvEnabled || false,
   });
 
   const mimeType = file.file.mimeType;

diff --git a/connectors/src/connectors/microsoft/temporal/mime_types.ts b/connectors/src/connectors/microsoft/temporal/mime_types.ts
@@ -1,12 +1,9 @@
-import { getEnabledFeatureFlagsMemoized } from "@connectors/lib/workspace";
-import type { ConnectorResource } from "@connectors/resources/connector_resource";
-
 export async function getMimeTypesToSync({
   pdfEnabled,
-  connector,
+  csvEnabled,
 }: {
   pdfEnabled: boolean;
-  connector: ConnectorResource;
+  csvEnabled: boolean;
 }) {
   const mimeTypes = [
     "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
@@ -17,16 +14,10 @@ export async function getMimeTypesToSync({
   if (pdfEnabled) {
     mimeTypes.push("application/pdf");
   }
-  const csvEnabled = await isCsvEnabled(connector);
   if (csvEnabled) {
     mimeTypes.push("application/vnd.ms-excel"); // Microsoft type for "text/csv"
     mimeTypes.push("text/csv");
   }
 
   return mimeTypes;
 }
-
-async function isCsvEnabled(connector: ConnectorResource): Promise<boolean> {
-  const enabledFeatureFlags = await getEnabledFeatureFlagsMemoized(connector);
-  return !!enabledFeatureFlags.includes("microsoft_csv_sync");
-}
diff --git a/connectors/src/lib/models/google_drive.ts b/connectors/src/lib/models/google_drive.ts
@@ -18,6 +18,7 @@ export class GoogleDriveConfig extends Model<
   declare updatedAt: CreationOptional<Date>;
   declare connectorId: ForeignKey<ConnectorModel["id"]>;
   declare pdfEnabled: boolean;
+  declare csvEnabled: boolean;
   declare largeFilesEnabled: boolean;
 }
 GoogleDriveConfig.init(
@@ -46,6 +47,11 @@ GoogleDriveConfig.init(
       allowNull: false,
       defaultValue: false,
     },
+    csvEnabled: {
+      type: DataTypes.BOOLEAN,
+      allowNull: false,
+      defaultValue: false,
+    },
     largeFilesEnabled: {
       type: DataTypes.BOOLEAN,
       allowNull: false,

diff --git a/connectors/src/lib/models/microsoft.ts b/connectors/src/lib/models/microsoft.ts
@@ -19,6 +19,7 @@ export class MicrosoftConfigurationModel extends Model<
   declare updatedAt: CreationOptional<Date>;
   declare connectorId: ForeignKey<ConnectorModel["id"]>;
   declare pdfEnabled: boolean;
+  declare csvEnabled: boolean;
   declare largeFilesEnabled: boolean;
 }
 MicrosoftConfigurationModel.init(
@@ -47,6 +48,11 @@ MicrosoftConfigurationModel.init(
       allowNull: false,
       defaultValue: false,
     },
+    csvEnabled: {
+      type: DataTypes.BOOLEAN,
+      allowNull: false,
+      defaultValue: false,
+    },
     largeFilesEnabled: {
       type: DataTypes.BOOLEAN,
       allowNull: false,

diff --git a/front/pages/poke/[wId]/data_sources/[name]/index.tsx b/front/pages/poke/[wId]/data_sources/[name]/index.tsx
@@ -44,6 +44,8 @@ type FeaturesType = {
   googleDriveLargeFilesEnabled: boolean;
   microsoftPdfEnabled: boolean;
   microsoftLargeFilesEnabled: boolean;
+  googleDriveCsvEnabled: boolean;
+  microsoftCsvEnabled: boolean;
   githubCodeSyncEnabled: boolean;
   autoReadChannelPattern: string | null;
 };
@@ -112,6 +114,8 @@ export const getServerSideProps = withSuperUserAuthRequirements<{
     googleDriveLargeFilesEnabled: false,
     microsoftPdfEnabled: false,
     microsoftLargeFilesEnabled: false,
+    googleDriveCsvEnabled: false,
+    microsoftCsvEnabled: false,
     githubCodeSyncEnabled: false,
     autoReadChannelPattern: null,
   };
@@ -153,6 +157,16 @@ export const getServerSideProps = withSuperUserAuthRequirements<{
         features.googleDrivePdfEnabled =
           gdrivePdfEnabledRes.value.configValue === "true";
 
+        const gdriveCsvEnabledRes = await connectorsAPI.getConnectorConfig(
+          dataSource.connectorId,
+          "csvEnabled"
+        );
+        if (gdriveCsvEnabledRes.isErr()) {
+          throw gdriveCsvEnabledRes.error;
+        }
+        features.googleDriveCsvEnabled =
+          gdriveCsvEnabledRes.value.configValue === "true";
+
         const gdriveLargeFilesEnabledRes =
           await connectorsAPI.getConnectorConfig(
             dataSource.connectorId,
@@ -175,6 +189,16 @@ export const getServerSideProps = withSuperUserAuthRequirements<{
         features.microsoftPdfEnabled =
           microsoftPdfEnabledRes.value.configValue === "true";
 
+        const microsoftCsvEnabledRes = await connectorsAPI.getConnectorConfig(
+          dataSource.connectorId,
+          "csvEnabled"
+        );
+        if (microsoftCsvEnabledRes.isErr()) {
+          throw microsoftCsvEnabledRes.error;
+        }
+        features.microsoftCsvEnabled =
+          microsoftCsvEnabledRes.value.configValue === "true";
+
         const microsoftLargeFilesEnabledRes =
           await connectorsAPI.getConnectorConfig(
             dataSource.connectorId,
@@ -339,6 +363,14 @@ const DataSourcePage = ({
                 configKey="pdfEnabled"
                 featureKey="googleDrivePdfEnabled"
               />
+              <ConfigToggle
+                title="CSV syncing enabled?"
+                owner={owner}
+                features={features}
+                dataSource={dataSource.name}
+                configKey="csvEnabled"
+                featureKey="googleDriveCsvEnabled"
+              />
 
               <ConfigToggle
                 title="Large Files enabled?"
@@ -360,6 +392,14 @@ const DataSourcePage = ({
                 configKey="pdfEnabled"
                 featureKey="microsoftPdfEnabled"
               />
+              <ConfigToggle
+                title="CSV syncing enabled?"
+                owner={owner}
+                features={features}
+                dataSource={dataSource.name}
+                configKey="csvEnabled"
+                featureKey="microsoftCsvEnabled"
+              />
               <ConfigToggle
                 title="Large Files enabled?"
                 owner={owner}

diff --git a/types/src/shared/feature_flags.ts b/types/src/shared/feature_flags.ts
@@ -7,8 +7,6 @@ export const WHITELISTABLE_FEATURES = [
   "microsoft_connector",
   "visualization_action_flag",
   "dust_splitted_ds_flag",
-  "microsoft_csv_sync",
-  "google_csv_sync",
   "test_oauth_setup",
 ] as const;
 export type WhitelistableFeature = (typeof WHITELISTABLE_FEATURES)[number];