From e529050bdd30c06a94e0d71e44e910c6cc549ca9 Mon Sep 17 00:00:00 2001 From: Jules Belveze <32683010+JulesBelveze@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:55:16 +0200 Subject: [PATCH] [connectors] - refacto: CSV enabled toggle (#6567) * [connectors] - feature: support CSV file sync for Microsoft connector - Replace connector parameter with csvEnabled to directly accept CSV sync configuration - Remove dependency of isCsvEnabled function by including csvEnabled in the configuration model [front] - feature: display CSV file sync configuration in UI - Add front-end support for toggling CSV file sync for both Google Drive and Microsoft data sources - Retrieve CSV sync configuration state from the back-end to reflect the current settings in the UI * [connectors] - feature: add CSV file support in Google Drive configuration - Introduce a new configuration option to enable processing of CSV files within the Google Drive connector - Default the csvEnabled flag to false, keeping CSV processing disabled by default * [connectors] - feature: add csvEnabled column to configuration tables - Added 'csvEnabled' column with a default value of false to Microsoft and Google Drive configuration tables to support CSV functionality. * [connectors] - feature: add support for CSV syncing in Google Drive and Microsoft connectors - Renamed migration file for consistency in numbering - Implemented CSV enable/disable toggles and handling in Google Drive connector - Added CSV sync configuration logic and added to workflows in Microsoft connector - Removed the old feature flags for CSV synchronization from the codebase [front] - feature: introduce user-facing toggles for CSV file synchronization - Added CSV sync toggles on data source configuration pages for Google Drive and Microsoft connectors [types] - refactor: remove CSV sync related feature flags - Deleted the old CSV sync feature flags 'microsoft_csv_sync' and 'google_csv_sync' from WhitelistableFeature type * [connectors] - feature: add csvEnabled configuration to Google Drive - Enabled the configuration for csv files support in Google Drive integration - Ensured the database migration reflects the addition of the csvEnabled field in the corresponding table --------- Co-authored-by: Jules --- .../20231109_2_create_gdrive_config.ts | 1 + connectors/migrations/db/migration_09.sql | 3 ++ .../src/connectors/google_drive/index.ts | 18 ++++++++- .../google_drive/temporal/activities.ts | 4 +- .../connectors/google_drive/temporal/file.ts | 2 +- .../google_drive/temporal/mime_types.ts | 18 +++------ connectors/src/connectors/microsoft/index.ts | 16 ++++++++ .../microsoft/temporal/activities.ts | 2 +- .../src/connectors/microsoft/temporal/file.ts | 2 +- .../microsoft/temporal/mime_types.ts | 13 +----- connectors/src/lib/models/google_drive.ts | 6 +++ connectors/src/lib/models/microsoft.ts | 6 +++ .../poke/[wId]/data_sources/[name]/index.tsx | 40 +++++++++++++++++++ types/src/shared/feature_flags.ts | 2 - 14 files changed, 101 insertions(+), 32 deletions(-) create mode 100644 connectors/migrations/db/migration_09.sql diff --git a/connectors/migrations/20231109_2_create_gdrive_config.ts b/connectors/migrations/20231109_2_create_gdrive_config.ts index f502be5f4cfe..ec3491a8b716 100644 --- a/connectors/migrations/20231109_2_create_gdrive_config.ts +++ b/connectors/migrations/20231109_2_create_gdrive_config.ts @@ -12,6 +12,7 @@ async function main() { const config = await GoogleDriveConfig.create({ connectorId: connector.id, pdfEnabled: false, + csvEnabled: false, largeFilesEnabled: false, }); console.log( diff --git a/connectors/migrations/db/migration_09.sql b/connectors/migrations/db/migration_09.sql new file mode 100644 index 000000000000..579bcf3d4ebe --- /dev/null +++ b/connectors/migrations/db/migration_09.sql @@ -0,0 +1,3 @@ +-- Migration created on Jul 29, 2024 +ALTER TABLE "public"."microsoft_configurations" ADD COLUMN "csvEnabled" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "public"."google_drive_configs" ADD COLUMN "csvEnabled" BOOLEAN NOT NULL DEFAULT false; \ No newline at end of file diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts index 02ce6ff0c8c5..7fecaa20ff65 100644 --- a/connectors/src/connectors/google_drive/index.ts +++ b/connectors/src/connectors/google_drive/index.ts @@ -117,6 +117,7 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { const googleDriveConfigurationBlob = { pdfEnabled: false, largeFilesEnabled: false, + csvEnabled: false, }; const connector = await ConnectorResource.makeNew( @@ -769,7 +770,19 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { } return new Ok(void 0); } - + case "csvEnabled": { + await config.update({ + csvEnabled: configValue === "true", + }); + const workflowRes = await launchGoogleDriveFullSyncWorkflow( + this.connectorId, + null + ); + if (workflowRes.isErr()) { + return workflowRes; + } + return new Ok(void 0); + } case "largeFilesEnabled": { await config.update({ largeFilesEnabled: configValue === "true", @@ -818,6 +831,9 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { case "largeFilesEnabled": { return new Ok(config.largeFilesEnabled ? "true" : "false"); } + case "csvEnabled": { + return new Ok(config.csvEnabled ? "true" : "false"); + } default: return new Err(new Error(`Invalid config key ${configKey}`)); } diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts index 1da758327463..b14e471191b2 100644 --- a/connectors/src/connectors/google_drive/temporal/activities.ts +++ b/connectors/src/connectors/google_drive/temporal/activities.ts @@ -166,7 +166,7 @@ export async function syncFiles( const mimeTypesToSync = await getMimeTypesToSync({ pdfEnabled: config?.pdfEnabled || false, - connector, + csvEnabled: config?.csvEnabled || false, }); const authCredentials = await getAuthObject(connector.connectionId); const driveFolder = await getGoogleDriveObject( @@ -351,7 +351,7 @@ export async function incrementalSync( }); const mimeTypesToSync = await getMimeTypesToSync({ pdfEnabled: config?.pdfEnabled || false, - connector, + csvEnabled: config?.csvEnabled || false, }); const selectedFoldersIds = await getFoldersToSync(connectorId); diff --git a/connectors/src/connectors/google_drive/temporal/file.ts b/connectors/src/connectors/google_drive/temporal/file.ts index 55ec1ee3e14e..36ee9b3067dc 100644 --- a/connectors/src/connectors/google_drive/temporal/file.ts +++ b/connectors/src/connectors/google_drive/temporal/file.ts @@ -182,7 +182,7 @@ export async function syncOneFile( const mimeTypesToDownload = await getMimeTypesToDownload({ pdfEnabled: config?.pdfEnabled || false, - connector, + csvEnabled: config?.csvEnabled || false, }); const documentId = getDocumentId(file.id); diff --git a/connectors/src/connectors/google_drive/temporal/mime_types.ts b/connectors/src/connectors/google_drive/temporal/mime_types.ts index 955ac3c5a894..da1d0f8189d7 100644 --- a/connectors/src/connectors/google_drive/temporal/mime_types.ts +++ b/connectors/src/connectors/google_drive/temporal/mime_types.ts @@ -1,6 +1,4 @@ import type { GoogleDriveFiles } from "@connectors/lib/models/google_drive"; -import { getEnabledFeatureFlagsMemoized } from "@connectors/lib/workspace"; -import type { ConnectorResource } from "@connectors/resources/connector_resource"; export const MIME_TYPES_TO_EXPORT: { [key: string]: string } = { "application/vnd.google-apps.document": "text/plain", @@ -9,10 +7,10 @@ export const MIME_TYPES_TO_EXPORT: { [key: string]: string } = { export async function getMimeTypesToDownload({ pdfEnabled, - connector, + csvEnabled, }: { pdfEnabled: boolean; - connector: ConnectorResource; + csvEnabled: boolean; }) { const mimeTypes = [ "text/plain", @@ -24,7 +22,6 @@ export async function getMimeTypesToDownload({ if (pdfEnabled) { mimeTypes.push("application/pdf"); } - const csvEnabled = await isCsvEnabled(connector); if (csvEnabled) { mimeTypes.push("text/csv"); } @@ -34,14 +31,14 @@ export async function getMimeTypesToDownload({ export async function getMimeTypesToSync({ pdfEnabled, - connector, + csvEnabled, }: { pdfEnabled: boolean; - connector: ConnectorResource; + csvEnabled: boolean; }) { const mimeTypes = await getMimeTypesToDownload({ pdfEnabled, - connector, + csvEnabled, }); mimeTypes.push(...Object.keys(MIME_TYPES_TO_EXPORT)); mimeTypes.push("application/vnd.google-apps.folder"); @@ -57,8 +54,3 @@ export function isGoogleDriveFolder(file: GoogleDriveFiles) { export function isGoogleDriveSpreadSheetFile(file: { mimeType: string }) { return file.mimeType === "application/vnd.google-apps.spreadsheet"; } - -async function isCsvEnabled(connector: ConnectorResource): Promise { - const enabledFeatureFlags = await getEnabledFeatureFlagsMemoized(connector); - return !!enabledFeatureFlags.includes("google_csv_sync"); -} diff --git a/connectors/src/connectors/microsoft/index.ts b/connectors/src/connectors/microsoft/index.ts index b5bc32b88d6c..053d892c49b1 100644 --- a/connectors/src/connectors/microsoft/index.ts +++ b/connectors/src/connectors/microsoft/index.ts @@ -78,6 +78,7 @@ export class MicrosoftConnectorManager extends BaseConnectorManager { const microsoftConfigurationBlob = { pdfEnabled: false, + csvEnabled: false, largeFilesEnabled: false, }; @@ -532,6 +533,18 @@ export class MicrosoftConnectorManager extends BaseConnectorManager { } return new Ok(undefined); } + case "csvEnabled": { + await config.update({ + csvEnabled: configValue === "true", + }); + const workflowRes = await launchMicrosoftFullSyncWorkflow( + this.connectorId + ); + if (workflowRes.isErr()) { + return workflowRes; + } + return new Ok(undefined); + } case "largeFilesEnabled": { await config.update({ @@ -578,6 +591,9 @@ export class MicrosoftConnectorManager extends BaseConnectorManager { case "pdfEnabled": { return new Ok(config.pdfEnabled ? "true" : "false"); } + case "csvEnabled": { + return new Ok(config.csvEnabled ? "true" : "false"); + } case "largeFilesEnabled": { return new Ok(config.largeFilesEnabled ? "true" : "false"); } diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts index 86f787fe77a2..53331cd4742d 100644 --- a/connectors/src/connectors/microsoft/temporal/activities.ts +++ b/connectors/src/connectors/microsoft/temporal/activities.ts @@ -334,7 +334,7 @@ export async function syncFiles({ const mimeTypesToSync = await getMimeTypesToSync({ pdfEnabled: providerConfig.pdfEnabled || false, - connector, + csvEnabled: providerConfig.csvEnabled || false, }); const filesToSync = children.filter( (item) => diff --git a/connectors/src/connectors/microsoft/temporal/file.ts b/connectors/src/connectors/microsoft/temporal/file.ts index 15df08eb090e..4a29888bfe26 100644 --- a/connectors/src/connectors/microsoft/temporal/file.ts +++ b/connectors/src/connectors/microsoft/temporal/file.ts @@ -125,7 +125,7 @@ export async function syncOneFile({ const mimeTypesToSync = await getMimeTypesToSync({ pdfEnabled: providerConfig.pdfEnabled || false, - connector, + csvEnabled: providerConfig.csvEnabled || false, }); const mimeType = file.file.mimeType; diff --git a/connectors/src/connectors/microsoft/temporal/mime_types.ts b/connectors/src/connectors/microsoft/temporal/mime_types.ts index 89602ae1b3c3..593ad621b2d1 100644 --- a/connectors/src/connectors/microsoft/temporal/mime_types.ts +++ b/connectors/src/connectors/microsoft/temporal/mime_types.ts @@ -1,12 +1,9 @@ -import { getEnabledFeatureFlagsMemoized } from "@connectors/lib/workspace"; -import type { ConnectorResource } from "@connectors/resources/connector_resource"; - export async function getMimeTypesToSync({ pdfEnabled, - connector, + csvEnabled, }: { pdfEnabled: boolean; - connector: ConnectorResource; + csvEnabled: boolean; }) { const mimeTypes = [ "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -17,7 +14,6 @@ export async function getMimeTypesToSync({ if (pdfEnabled) { mimeTypes.push("application/pdf"); } - const csvEnabled = await isCsvEnabled(connector); if (csvEnabled) { mimeTypes.push("application/vnd.ms-excel"); // Microsoft type for "text/csv" mimeTypes.push("text/csv"); @@ -25,8 +21,3 @@ export async function getMimeTypesToSync({ return mimeTypes; } - -async function isCsvEnabled(connector: ConnectorResource): Promise { - const enabledFeatureFlags = await getEnabledFeatureFlagsMemoized(connector); - return !!enabledFeatureFlags.includes("microsoft_csv_sync"); -} diff --git a/connectors/src/lib/models/google_drive.ts b/connectors/src/lib/models/google_drive.ts index 4ece7eda9e39..0ccf37f2eac9 100644 --- a/connectors/src/lib/models/google_drive.ts +++ b/connectors/src/lib/models/google_drive.ts @@ -18,6 +18,7 @@ export class GoogleDriveConfig extends Model< declare updatedAt: CreationOptional; declare connectorId: ForeignKey; declare pdfEnabled: boolean; + declare csvEnabled: boolean; declare largeFilesEnabled: boolean; } GoogleDriveConfig.init( @@ -46,6 +47,11 @@ GoogleDriveConfig.init( allowNull: false, defaultValue: false, }, + csvEnabled: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }, largeFilesEnabled: { type: DataTypes.BOOLEAN, allowNull: false, diff --git a/connectors/src/lib/models/microsoft.ts b/connectors/src/lib/models/microsoft.ts index 3e36731e2612..b11327b20f7d 100644 --- a/connectors/src/lib/models/microsoft.ts +++ b/connectors/src/lib/models/microsoft.ts @@ -19,6 +19,7 @@ export class MicrosoftConfigurationModel extends Model< declare updatedAt: CreationOptional; declare connectorId: ForeignKey; declare pdfEnabled: boolean; + declare csvEnabled: boolean; declare largeFilesEnabled: boolean; } MicrosoftConfigurationModel.init( @@ -47,6 +48,11 @@ MicrosoftConfigurationModel.init( allowNull: false, defaultValue: false, }, + csvEnabled: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }, largeFilesEnabled: { type: DataTypes.BOOLEAN, allowNull: false, diff --git a/front/pages/poke/[wId]/data_sources/[name]/index.tsx b/front/pages/poke/[wId]/data_sources/[name]/index.tsx index 9df51b002e75..0c069c36ca98 100644 --- a/front/pages/poke/[wId]/data_sources/[name]/index.tsx +++ b/front/pages/poke/[wId]/data_sources/[name]/index.tsx @@ -44,6 +44,8 @@ type FeaturesType = { googleDriveLargeFilesEnabled: boolean; microsoftPdfEnabled: boolean; microsoftLargeFilesEnabled: boolean; + googleDriveCsvEnabled: boolean; + microsoftCsvEnabled: boolean; githubCodeSyncEnabled: boolean; autoReadChannelPattern: string | null; }; @@ -112,6 +114,8 @@ export const getServerSideProps = withSuperUserAuthRequirements<{ googleDriveLargeFilesEnabled: false, microsoftPdfEnabled: false, microsoftLargeFilesEnabled: false, + googleDriveCsvEnabled: false, + microsoftCsvEnabled: false, githubCodeSyncEnabled: false, autoReadChannelPattern: null, }; @@ -153,6 +157,16 @@ export const getServerSideProps = withSuperUserAuthRequirements<{ features.googleDrivePdfEnabled = gdrivePdfEnabledRes.value.configValue === "true"; + const gdriveCsvEnabledRes = await connectorsAPI.getConnectorConfig( + dataSource.connectorId, + "csvEnabled" + ); + if (gdriveCsvEnabledRes.isErr()) { + throw gdriveCsvEnabledRes.error; + } + features.googleDriveCsvEnabled = + gdriveCsvEnabledRes.value.configValue === "true"; + const gdriveLargeFilesEnabledRes = await connectorsAPI.getConnectorConfig( dataSource.connectorId, @@ -175,6 +189,16 @@ export const getServerSideProps = withSuperUserAuthRequirements<{ features.microsoftPdfEnabled = microsoftPdfEnabledRes.value.configValue === "true"; + const microsoftCsvEnabledRes = await connectorsAPI.getConnectorConfig( + dataSource.connectorId, + "csvEnabled" + ); + if (microsoftCsvEnabledRes.isErr()) { + throw microsoftCsvEnabledRes.error; + } + features.microsoftCsvEnabled = + microsoftCsvEnabledRes.value.configValue === "true"; + const microsoftLargeFilesEnabledRes = await connectorsAPI.getConnectorConfig( dataSource.connectorId, @@ -339,6 +363,14 @@ const DataSourcePage = ({ configKey="pdfEnabled" featureKey="googleDrivePdfEnabled" /> + +