From 72f401fc82dc2327dca9ce78f47d4d5cfadbd7a5 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 18 Sep 2024 14:51:56 -0400 Subject: [PATCH 1/2] update event shape --- components/gmail/package.json | 2 +- .../new-email-received/new-email-received.mjs | 44 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/components/gmail/package.json b/components/gmail/package.json index 0b85aad615d11..a1cbf862e99a3 100644 --- a/components/gmail/package.json +++ b/components/gmail/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/gmail", - "version": "0.1.5", + "version": "0.1.6", "description": "Pipedream Gmail Components", "main": "gmail.app.mjs", "keywords": [ diff --git a/components/gmail/sources/new-email-received/new-email-received.mjs b/components/gmail/sources/new-email-received/new-email-received.mjs index 16977a23a0b0f..95c561e3b47c4 100644 --- a/components/gmail/sources/new-email-received/new-email-received.mjs +++ b/components/gmail/sources/new-email-received/new-email-received.mjs @@ -15,7 +15,7 @@ export default { name: "New Email Received", description: "Emit new event when a new email is received.", type: "source", - version: "0.1.2", + version: "0.1.3", dedupe: "unique", props: { gmail, @@ -366,14 +366,14 @@ export default { return messageDetails.map((msg) => { const headers = msg.payload.headers; return { - id: msg.id, - threadId: msg.threadId, - subject: headers.find((h) => h.name.toLowerCase() === "subject") - ?.value, - from: headers.find((h) => h.name.toLowerCase() === "from")?.value, - to: headers.find((h) => h.name.toLowerCase() === "to")?.value, - date: headers.find((h) => h.name.toLowerCase() === "date")?.value, - snippet: msg.snippet, + "id": msg.id, + "threadId": msg.threadId, + "subject": headers.find((h) => h.name.toLowerCase() === "subject")?.value, + "from": headers.find((h) => h.name.toLowerCase() === "from")?.value, + "to": headers.find((h) => h.name.toLowerCase() === "to")?.value, + "reply-to": headers.find((h) => h.name.toLowerCase() === "reply-to")?.value, + "date": headers.find((h) => h.name.toLowerCase() === "date")?.value, + "snippet": msg.snippet, }; }); }, @@ -482,6 +482,10 @@ export default { const newMessageIds = newMessages?.map(({ id }) => id) || []; const messageDetails = await this.gmail.getMessages(newMessageIds); + if (!messageDetails?.length) { + return; + } + console.log("Fetched message details count:", messageDetails.length); const processedEmails = this.processEmails(messageDetails); @@ -491,19 +495,15 @@ export default { this._setLastProcessedHistoryId(latestHistoryId); console.log("Updated lastProcessedHistoryId:", latestHistoryId); - if (processedEmails?.length) { - this.$emit( - { - newEmailsCount: processedEmails.length, - emails: processedEmails, - lastProcessedHistoryId: latestHistoryId, - }, - { - id: processedEmails[0].id, - summary: processedEmails[0].subject, - ts: Date.now(), - }, - ); + for (let i = 0; i < messageDetails.length; i++) { + this.$emit({ + ...messageDetails[i], + "parsed-headers": processedEmails[i], + }, { + id: processedEmails[i].id, + summary: processedEmails[i].subject, + ts: Date.now(), + }); } } }, From d1a05fdb4874dd2c81dde3e2b49dc8af2d6808ac Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Thu, 19 Sep 2024 12:03:48 -0400 Subject: [PATCH 2/2] updates --- components/gmail/sources/common/base.mjs | 18 +++++++++++ .../gmail/sources/common/polling-history.mjs | 3 -- .../new-attachment-received.mjs | 4 +-- .../new-email-matching-search.mjs | 4 +-- .../new-email-received/new-email-received.mjs | 31 ++----------------- .../new-labeled-email/new-labeled-email.mjs | 4 +-- .../sources/new-sent-email/new-sent-email.mjs | 4 +-- 7 files changed, 28 insertions(+), 40 deletions(-) diff --git a/components/gmail/sources/common/base.mjs b/components/gmail/sources/common/base.mjs index cd7a3d19bc9cd..e6339017cb0d1 100644 --- a/components/gmail/sources/common/base.mjs +++ b/components/gmail/sources/common/base.mjs @@ -44,7 +44,25 @@ export default { decodedContent: Buffer.from(firstPart.body.data, "base64").toString(), }; }, + processEmail(msg) { + // Process and structure the email data + const headers = msg.payload.headers; + return { + "id": msg.id, + "threadId": msg.threadId, + "subject": headers.find((h) => h.name.toLowerCase() === "subject")?.value, + "from": headers.find((h) => h.name.toLowerCase() === "from")?.value, + "to": headers.find((h) => h.name.toLowerCase() === "to")?.value, + "reply-to": headers.find((h) => h.name.toLowerCase() === "reply-to")?.value, + "date": headers.find((h) => h.name.toLowerCase() === "date")?.value, + "snippet": msg.snippet, + }; + }, emitEvent(message) { + message = { + ...message, + parsedHeaders: this.processEmail(message), + }; const meta = this.generateMeta(message); this.$emit(this.decodeContent(message), meta); }, diff --git a/components/gmail/sources/common/polling-history.mjs b/components/gmail/sources/common/polling-history.mjs index 9fc8ae1f0a7b7..b781e990a80e3 100644 --- a/components/gmail/sources/common/polling-history.mjs +++ b/components/gmail/sources/common/polling-history.mjs @@ -6,9 +6,6 @@ export default { hooks: { ...common.hooks, async deploy() { - if (this.triggerType === "webhook") { - return; - } const historyId = await this.getHistoryId(); if (!historyId) { return; diff --git a/components/gmail/sources/new-attachment-received/new-attachment-received.mjs b/components/gmail/sources/new-attachment-received/new-attachment-received.mjs index ac831e1f5730e..b959c827df0f5 100644 --- a/components/gmail/sources/new-attachment-received/new-attachment-received.mjs +++ b/components/gmail/sources/new-attachment-received/new-attachment-received.mjs @@ -6,7 +6,7 @@ export default { key: "gmail-new-attachment-received", name: "New Attachment Received", description: "Emit new event for each attachment in a message received. This source is capped at 100 max new messages per run.", - version: "0.0.3", + version: "0.0.4", type: "source", dedupe: "unique", props: { @@ -51,7 +51,7 @@ export default { return { id: `${message.id}${attachment.partId}`, summary: `New Attachment: ${attachment.filename}`, - ts: message.internalDate, + ts: +message.internalDate, }; }, emitEvent(message) { diff --git a/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs b/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs index aa5de10dac48d..9ae4da9f2d357 100644 --- a/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs +++ b/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs @@ -6,7 +6,7 @@ export default { key: "gmail-new-email-matching-search", name: "New Email Matching Search", description: "Emit new event when an email matching the search criteria is received. This source is capped at 100 max new messages per run.", - version: "0.0.2", + version: "0.0.3", type: "source", dedupe: "unique", props: { @@ -39,7 +39,7 @@ export default { return { id: message.id, summary: `New email: ${subject}`, - ts: message.internalDate, + ts: +message.internalDate, }; }, }, diff --git a/components/gmail/sources/new-email-received/new-email-received.mjs b/components/gmail/sources/new-email-received/new-email-received.mjs index 95c561e3b47c4..f422dce4c5d6f 100644 --- a/components/gmail/sources/new-email-received/new-email-received.mjs +++ b/components/gmail/sources/new-email-received/new-email-received.mjs @@ -361,22 +361,6 @@ export default { } return topic; }, - processEmails(messageDetails) { - // Process and structure the email data - return messageDetails.map((msg) => { - const headers = msg.payload.headers; - return { - "id": msg.id, - "threadId": msg.threadId, - "subject": headers.find((h) => h.name.toLowerCase() === "subject")?.value, - "from": headers.find((h) => h.name.toLowerCase() === "from")?.value, - "to": headers.find((h) => h.name.toLowerCase() === "to")?.value, - "reply-to": headers.find((h) => h.name.toLowerCase() === "reply-to")?.value, - "date": headers.find((h) => h.name.toLowerCase() === "date")?.value, - "snippet": msg.snippet, - }; - }); - }, getHistoryTypes() { return [ "messageAdded", @@ -386,7 +370,7 @@ export default { return { id: message.id, summary: message.snippet, - ts: message.internalDate, + ts: +message.internalDate, }; }, filterHistory(history) { @@ -488,23 +472,12 @@ export default { console.log("Fetched message details count:", messageDetails.length); - const processedEmails = this.processEmails(messageDetails); - // Store the latest historyId in the db const latestHistoryId = historyResponse.historyId || receivedHistoryId; this._setLastProcessedHistoryId(latestHistoryId); console.log("Updated lastProcessedHistoryId:", latestHistoryId); - for (let i = 0; i < messageDetails.length; i++) { - this.$emit({ - ...messageDetails[i], - "parsed-headers": processedEmails[i], - }, { - id: processedEmails[i].id, - summary: processedEmails[i].subject, - ts: Date.now(), - }); - } + messageDetails.forEach((message) => this.emitEvent(message)); } }, }; diff --git a/components/gmail/sources/new-labeled-email/new-labeled-email.mjs b/components/gmail/sources/new-labeled-email/new-labeled-email.mjs index 0a186606d37da..6b0aaa859b12a 100644 --- a/components/gmail/sources/new-labeled-email/new-labeled-email.mjs +++ b/components/gmail/sources/new-labeled-email/new-labeled-email.mjs @@ -8,7 +8,7 @@ export default { name: "New Labeled Email", description: "Emit new event when a new email is labeled.", type: "source", - version: "0.0.3", + version: "0.0.4", dedupe: "unique", props: { ...common.props, @@ -32,7 +32,7 @@ export default { return { id: `${message.id}-${this.label}`, summary: `A new message with ID: ${message.id} was labeled with "${this.label}"`, - ts: Date.now(), + ts: +message.internalDate, }; }, filterHistory(history) { diff --git a/components/gmail/sources/new-sent-email/new-sent-email.mjs b/components/gmail/sources/new-sent-email/new-sent-email.mjs index 8c2d9712c7173..3ef53b25aac53 100644 --- a/components/gmail/sources/new-sent-email/new-sent-email.mjs +++ b/components/gmail/sources/new-sent-email/new-sent-email.mjs @@ -6,7 +6,7 @@ export default { key: "gmail-new-sent-email", name: "New Sent Email", description: "Emit new event for each new email sent. (Maximum of 100 events emited per execution)", - version: "0.0.3", + version: "0.0.4", type: "source", dedupe: "unique", props: { @@ -30,7 +30,7 @@ export default { return { id: message.id, summary: message.snippet, - ts: new Date(message.internalDate), + ts: +message.internalDate, }; }, },