diff --git a/_emulator/.firebaserc b/_emulator/.firebaserc index 5d2802a3c2..8019a3d996 100644 --- a/_emulator/.firebaserc +++ b/_emulator/.firebaserc @@ -1,5 +1,13 @@ { "projects": { "default": "demo-test" + }, + "targets": {}, + "etags": { + "dev-extensions-testing": { + "extensionInstances": { + "firestore-bigquery-export": "02acbd8b443b9635716d52d65758a78db1e51140191caecaaf60d932d314a62a" + } + } } } \ No newline at end of file diff --git a/_emulator/firebase.json b/_emulator/firebase.json index 70e56c96de..c085b9180b 100644 --- a/_emulator/firebase.json +++ b/_emulator/firebase.json @@ -1,11 +1,6 @@ { "extensions": { - "firestore-send-email": "../firestore-send-email", - "delete-user-data": "../delete-user-data", - "storage-resize-images": "../storage-resize-images", - "firestore-counter": "../firestore-counter", - "firestore-bigquery-export": "../firestore-bigquery-export", - "firestore-send-email-sendgrid": "../firestore-send-email" + "firestore-bigquery-export": "../firestore-bigquery-export" }, "storage": { "rules": "storage.rules" diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index a8a604d3c8..e41c68043e 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -196,19 +196,6 @@ params: default: posts required: true - - param: LOG_FAILED_EXPORTS - label: Enable logging failed exports - description: >- - If enabled, the extension will log event exports that failed to enqueue to - Cloud Logging, to mitigate data loss. - type: select - options: - - label: Yes - value: yes - - label: No - value: no - default: yes - - param: WILDCARD_IDS label: Enable Wildcard Column field with Parent Firestore Document IDs description: >- diff --git a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap index b9cb5f5417..8d6803f5b6 100644 --- a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap +++ b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap @@ -21,7 +21,6 @@ Object { "instanceId": undefined, "kmsKeyName": "test", "location": "us-central1", - "logFailedExportData": false, "maxDispatchesPerSecond": 10, "maxEnqueueAttempts": 3, "tableId": "my_table", diff --git a/firestore-bigquery-export/functions/__tests__/e2e.test.ts b/firestore-bigquery-export/functions/__tests__/e2e.test.ts index 48851c7229..4773a3e25b 100644 --- a/firestore-bigquery-export/functions/__tests__/e2e.test.ts +++ b/firestore-bigquery-export/functions/__tests__/e2e.test.ts @@ -3,8 +3,8 @@ import { BigQuery } from "@google-cloud/bigquery"; /** Set defaults */ const bqProjectId = process.env.BQ_PROJECT_ID || "dev-extensions-testing"; -const datasetId = process.env.DATASET_ID || "firestore_export_e2e"; -const tableId = process.env.TABLE_ID || "posts_raw_changelog"; +const datasetId = process.env.DATASET_ID || "firestore_export"; +const tableId = process.env.TABLE_ID || "bq_e2e_test_raw_changelog"; /** Init resources */ admin.initializeApp({ projectId: bqProjectId }); @@ -34,7 +34,7 @@ describe("e2e", () => { /** Get the latest record from this table */ const [changeLogQuery] = await bq.createQueryJob({ - query: `SELECT * FROM \`${bqProjectId}.${datasetId}.${tableId}\` ORDER BY timestamp DESC \ LIMIT 1`, + query: `SELECT * FROM \`${bqProjectId}.${datasetId}.${tableId}\` ORDER BY timestamp DESC LIMIT 1`, }); const [rows] = await changeLogQuery.getQueryResults(); diff --git a/firestore-bigquery-export/functions/__tests__/functions.test.ts b/firestore-bigquery-export/functions/__tests__/functions.test.ts index 801ef71dcd..d9aefb52e8 100644 --- a/firestore-bigquery-export/functions/__tests__/functions.test.ts +++ b/firestore-bigquery-export/functions/__tests__/functions.test.ts @@ -37,6 +37,7 @@ jest.mock("firebase-admin/functions", () => ({ })); jest.mock("../src/logs", () => ({ + ...jest.requireActual("../src/logs"), start: jest.fn(() => logger.log("Started execution of extension with configuration", config) ), diff --git a/firestore-bigquery-export/functions/firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz b/firestore-bigquery-export/functions/firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz deleted file mode 100644 index 44c71a9536..0000000000 Binary files a/firestore-bigquery-export/functions/firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz and /dev/null differ diff --git a/firestore-bigquery-export/functions/package-lock.json b/firestore-bigquery-export/functions/package-lock.json index b32764c859..c4a9ef1457 100644 --- a/firestore-bigquery-export/functions/package-lock.json +++ b/firestore-bigquery-export/functions/package-lock.json @@ -574,7 +574,7 @@ "node_modules/@firebaseextensions/firestore-bigquery-change-tracker": { "version": "1.1.37", "resolved": "file:firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz", - "integrity": "sha512-+pepcVgtXurbgLjHyDz/fWWNrThAa+UANY+1+kfBRr6V+AzS7wrtSyCRO5bfbO1L/0Tn3DHENJVCWBqeMcwTyw==", + "integrity": "sha512-CojXoQch6TPZgWOt2Fikb4aVHTETUloVhCx9/S+1c2+0aHBhltvzwFbCxPvkWf4Cr7a/6CA8e771WvR2lVLXEQ==", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0", @@ -2732,8 +2732,9 @@ "license": "MIT" }, "node_modules/cookie": { - "version": "0.6.0", - "license": "MIT", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "engines": { "node": ">= 0.6" } @@ -3403,16 +3404,16 @@ } }, "node_modules/express": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", - "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", diff --git a/firestore-bigquery-export/functions/package.json b/firestore-bigquery-export/functions/package.json index cec788c6a3..5450224fd3 100644 --- a/firestore-bigquery-export/functions/package.json +++ b/firestore-bigquery-export/functions/package.json @@ -13,7 +13,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "file:firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.38", "@google-cloud/bigquery": "^7.6.0", "@types/chai": "^4.1.6", "@types/express-serve-static-core": "4.17.30", diff --git a/firestore-bigquery-export/functions/src/config.ts b/firestore-bigquery-export/functions/src/config.ts index 15134fc69e..9e4b612198 100644 --- a/firestore-bigquery-export/functions/src/config.ts +++ b/firestore-bigquery-export/functions/src/config.ts @@ -34,7 +34,6 @@ export function clustering(clusters: string | undefined) { } export default { - logFailedExportData: process.env.LOG_FAILED_EXPORTS === "yes", bqProjectId: process.env.BIGQUERY_PROJECT_ID, databaseId: "(default)", collectionPath: process.env.COLLECTION_PATH, diff --git a/firestore-bigquery-export/functions/src/index.ts b/firestore-bigquery-export/functions/src/index.ts index 48ba3e6902..13e1b577a1 100644 --- a/firestore-bigquery-export/functions/src/index.ts +++ b/firestore-bigquery-export/functions/src/index.ts @@ -22,13 +22,12 @@ import { getFunctions } from "firebase-admin/functions"; import { ChangeType, FirestoreBigQueryEventHistoryTracker, - FirestoreEventHistoryTracker, + FirestoreDocumentChangeEvent, } from "@firebaseextensions/firestore-bigquery-change-tracker"; import * as logs from "./logs"; import * as events from "./events"; -import { getChangeType, getDocumentId, resolveWildcardIds } from "./util"; -import { backupToGCS } from "./cloud_storage_backups"; +import { getChangeType, getDocumentId } from "./util"; // Configuration for the Firestore Event History Tracker. const eventTrackerConfig = { @@ -53,7 +52,7 @@ const eventTrackerConfig = { }; // Initialize the Firestore Event History Tracker with the given configuration. -const eventTracker: FirestoreEventHistoryTracker = +const eventTracker: FirestoreBigQueryEventHistoryTracker = new FirestoreBigQueryEventHistoryTracker(eventTrackerConfig); // Initialize logging. @@ -74,6 +73,17 @@ export const syncBigQuery = functions.tasks .taskQueue() .onDispatch( async ({ context, changeType, documentId, data, oldData }, ctx) => { + const documentName = context.resource.name; + const eventId = context.eventId; + const operation = changeType; + + logs.logEventAction( + "Firestore event received by onDispatch trigger", + documentName, + eventId, + operation + ); + try { // Use the shared function to write the event to BigQuery await recordEventToBigQuery( @@ -103,24 +113,13 @@ export const syncBigQuery = functions.tasks logs.complete(); } catch (err) { // Log error and throw it to handle in the calling function. - logs.error(true, "Failed to process syncBigQuery task", err, { - context, - changeType, - documentId, - data, - oldData, - }); - - // if (config.backupToGCS) { - // // Backup to Google Cloud Storage as a last resort. - // await backupToGCS(config.backupBucketName, config.backupDir, { - // changeType, - // documentId, - // serializedData: data, - // serializedOldData: oldData, - // context, - // }); - // } + logs.logFailedEventAction( + "Failed to write event to BigQuery from onDispatch handler", + documentName, + eventId, + operation, + err as Error + ); throw err; } @@ -150,6 +149,17 @@ export const fsexportbigquery = functions.firestore const oldData = isCreated || config.excludeOldData ? undefined : change.before?.data(); + const documentName = context.resource.name; + const eventId = context.eventId; + const operation = changeType; + + logs.logEventAction( + "Firestore event received by onWrite trigger", + documentName, + eventId, + operation + ); + let serializedData: any; let serializedOldData: any; @@ -159,7 +169,13 @@ export const fsexportbigquery = functions.firestore serializedOldData = eventTracker.serializeData(oldData); } catch (err) { // Log serialization error and throw it. - logs.error(true, "Failed to serialize data", err, { data, oldData }); + logs.logFailedEventAction( + "Failed to serialize data", + documentName, + eventId, + operation, + err as Error + ); throw err; } @@ -217,18 +233,20 @@ export const fsexportbigquery = functions.firestore * @param context - The event context from Firestore. */ async function recordEventToBigQuery( - changeType: string, + changeType: ChangeType, documentId: string, serializedData: any, serializedOldData: any, context: functions.EventContext ) { - const event = { + const event: FirestoreDocumentChangeEvent = { timestamp: context.timestamp, // Cloud Firestore commit timestamp operation: changeType, // The type of operation performed documentName: context.resource.name, // The document name documentId, // The document ID - pathParams: config.wildcardIds ? context.params : null, // Path parameters, if any + pathParams: (config.wildcardIds ? context.params : null) as + | FirestoreDocumentChangeEvent["pathParams"] + | null, // Path parameters, if any eventId: context.eventId, // The event ID from Firestore data: serializedData, // Serialized new data oldData: serializedOldData, // Serialized old data @@ -251,7 +269,7 @@ async function recordEventToBigQuery( async function attemptToEnqueue( err: Error, context: functions.EventContext, - changeType: string, + changeType: ChangeType, documentId: string, serializedData: any, serializedOldData: any @@ -295,40 +313,21 @@ async function attemptToEnqueue( } } catch (enqueueErr) { // Prepare the event object for error logging. - const event = { - timestamp: context.timestamp, - operation: changeType, - documentName: context.resource.name, - documentId, - pathParams: config.wildcardIds ? context.params : null, - eventId: context.eventId, - data: serializedData, - oldData: serializedOldData, - }; // Record the error event. await events.recordErrorEvent(enqueueErr as Error); - // Log the error if it has not been logged already. - if (!enqueueErr.logged && config.logFailedExportData) { - logs.error( - true, - "Failed to enqueue task to syncBigQuery", - enqueueErr, - event - ); - } + const documentName = context.resource.name; + const eventId = context.eventId; + const operation = changeType; - // if (config.backupToGCS) { - // // Backup to Google Cloud Storage as a last resort. - // await backupToGCS(config.backupBucketName, config.backupDir, { - // changeType, - // documentId, - // serializedData, - // serializedOldData, - // context, - // }); - // } + logs.logFailedEventAction( + "Failed to enqueue event to Cloud Tasks from onWrite handler", + documentName, + eventId, + operation, + enqueueErr as Error + ); } } diff --git a/firestore-bigquery-export/functions/src/logs.ts b/firestore-bigquery-export/functions/src/logs.ts index d81ee7fb08..5eb112f3e9 100644 --- a/firestore-bigquery-export/functions/src/logs.ts +++ b/firestore-bigquery-export/functions/src/logs.ts @@ -15,6 +15,7 @@ */ import { logger } from "firebase-functions"; import config from "./config"; +import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker"; export const arrayFieldInvalid = (fieldName: string) => { logger.warn(`Array field '${fieldName}' does not contain an array, skipping`); @@ -182,3 +183,31 @@ export const timestampMissingValue = (fieldName: string) => { `Missing value for timestamp field: ${fieldName}, using default timestamp instead.` ); }; + +export const logEventAction = ( + action: string, + document_name: string, + event_id: string, + operation: ChangeType +) => { + logger.info(action, { + document_name, + event_id, + operation, + }); +}; + +export const logFailedEventAction = ( + action: string, + document_name: string, + event_id: string, + operation: ChangeType, + error: Error +) => { + logger.error(action, { + document_name, + event_id, + operation, + error, + }); +};