Skip to content

Commit

Permalink
Aric revert gdrive double pass (#2432)
Browse files Browse the repository at this point in the history
* Revert "Fix Google Drive Webhook double pass (#2403)"

This reverts commit b046c18.

* Revert "Double pass on Gdrive webhook (#2398)"

This reverts commit 8a343be.

* Debounce over 30 seconds (vs 10)

* Addressing @spolu's comments
  • Loading branch information
lasryaric authored Nov 8, 2023
1 parent 551d4d2 commit 78c0b6e
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 28 deletions.
6 changes: 1 addition & 5 deletions connectors/src/connectors/google_drive/temporal/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,7 @@ export async function launchGoogleDriveIncrementalSyncWorkflow(
const workflowId = googleDriveIncrementalSyncWorkflowId(connectorId);
try {
await client.workflow.signalWithStart(googleDriveIncrementalSync, {
args: [
connectorIdModelId,
dataSourceConfig,
{ no: 2, delay: 60 * 5 * 1000 },
],
args: [connectorIdModelId, dataSourceConfig],
taskQueue: QUEUE_NAME,
workflowId: workflowId,
signal: newWebhookSignal,
Expand Down
2 changes: 1 addition & 1 deletion connectors/src/connectors/google_drive/temporal/config.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export const WORKFLOW_VERSION = 2;
export const WORKFLOW_VERSION = 3;
export const QUEUE_NAME = `google-queue-v${WORKFLOW_VERSION}`;
25 changes: 3 additions & 22 deletions connectors/src/connectors/google_drive/temporal/workflows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,17 +106,8 @@ export function googleDriveFullSyncWorkflowId(connectorId: string) {

export async function googleDriveIncrementalSync(
connectorId: ModelId,
dataSourceConfig: DataSourceConfig,
pass: {
// Number of times to run the incremental sync workflow.
// There is sometimes a delay between the time we receive a Google Drive webhook
// and the time the changes are actually available in the Gdrive API.
// So we run the incremental sync workflow multiple times to increase the chances of catching the changes.
no: number;
delay: number;
}
dataSourceConfig: DataSourceConfig
) {
pass.no = pass.no - 1;
let signaled = false;
let debounceCount = 0;
setHandler(newWebhookSignal, () => {
Expand Down Expand Up @@ -148,19 +139,9 @@ export async function googleDriveIncrementalSync(
);
} while (nextPageToken);
}
}

await syncSucceeded(connectorId);
console.log("googleDriveIncrementalSync done for connectorId", connectorId);
if (pass.no > 0) {
await sleep(pass.delay);
await executeChild(googleDriveIncrementalSync, {
workflowId: `${workflowInfo().workflowId}-pass-${
pass.no
}-${new Date().getTime()}`,
args: [connectorId, dataSourceConfig, pass],
memo: workflowInfo().memo,
});
await syncSucceeded(connectorId);
console.log("googleDriveIncrementalSync done for connectorId", connectorId);
}
}

Expand Down

0 comments on commit 78c0b6e

Please sign in to comment.