-
Notifications
You must be signed in to change notification settings - Fork 384
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore(firestore-bigquery-export): temporarily disable GCS
- Loading branch information
Showing
14 changed files
with
262 additions
and
63 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file added
BIN
+13.9 KB
...bigquery-export/functions/firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz
Binary file not shown.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,7 +13,7 @@ | |
| "author": "Jan Wyszynski <[email protected]>", | ||
| "license": "Apache-2.0", | ||
| "dependencies": { | ||
| "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.37", | ||
| "@firebaseextensions/firestore-bigquery-change-tracker": "file:firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz", | ||
| "@google-cloud/bigquery": "^7.6.0", | ||
| "@types/chai": "^4.1.6", | ||
| "@types/express-serve-static-core": "4.17.30", | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| const admin = require("firebase-admin"); | ||
|
|
||
| // Initialize Firebase Admin with your credentials | ||
| // Make sure you've already set up your Firebase Admin SDK | ||
| admin.initializeApp({ | ||
| projectId: "vertex-testing-1efc3", | ||
| }); | ||
|
|
||
| const firestore = admin.firestore(); | ||
|
|
||
| async function countDocuments(collectionPath) { | ||
| try { | ||
| const collectionRef = firestore.collection(collectionPath); | ||
|
|
||
| // Perform an aggregate query to count the documents | ||
| const snapshot = await collectionRef.count().get(); | ||
|
|
||
| // Access the count from the snapshot | ||
| const docCount = snapshot.data().count; | ||
|
|
||
| console.log( | ||
| `Number of documents in collection '${collectionPath}':`, | ||
| docCount | ||
| ); | ||
| return docCount; | ||
| } catch (error) { | ||
| console.error("Error counting documents:", error); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| // Call the function and pass the collection path | ||
| countDocuments("posts_2"); |
104 changes: 104 additions & 0 deletions
104
firestore-bigquery-export/functions/stress_test/main.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| const { Worker } = require("worker_threads"); | ||
| const { performance } = require("perf_hooks"); | ||
| const path = require("path"); | ||
|
|
||
| const totalDocs = 10000000; // Total number of documents to write | ||
| const maxThreads = 20; // Maximum number of worker threads | ||
| const batchSize = 500; // Documents per batch | ||
| const rampUpDelay = 2000; // 5 seconds delay between ramp-ups | ||
| const rampUps = 20; // Number of ramp-ups (planned) | ||
|
|
||
| const docsPerRampUp = Math.ceil(totalDocs / rampUps); // Documents per ramp-up | ||
|
|
||
| // Start measuring total execution time | ||
| const totalStartTime = performance.now(); | ||
|
|
||
| const workerJsPath = path.resolve(__dirname, "worker.js"); | ||
|
|
||
| // Function to spawn worker threads for a specific ramp-up | ||
| const spawnWorkers = async (activeThreads, startDoc, docsPerRampUp) => { | ||
| console.log(`Spawning ${activeThreads} worker(s)...`); | ||
| let promises = []; | ||
| const docsPerThread = Math.ceil(docsPerRampUp / activeThreads); | ||
|
|
||
| for (let i = 0; i < activeThreads; i++) { | ||
| const docsForThisThread = Math.min(docsPerThread, docsPerRampUp); | ||
| const start = startDoc + i * docsPerThread; | ||
| const end = Math.min(start + docsForThisThread, startDoc + docsPerRampUp); | ||
|
|
||
| promises.push( | ||
| new Promise((resolve, reject) => { | ||
| const worker = new Worker(workerJsPath, { | ||
| workerData: { | ||
| start, | ||
| end, | ||
| batchSize, | ||
| }, | ||
| }); | ||
|
|
||
| worker.on("message", (message) => { | ||
| console.log(`Worker ${i + 1}: ${message}`); | ||
| }); | ||
|
|
||
| worker.on("error", (err) => { | ||
| console.error(`Worker ${i + 1} error: ${err}`); | ||
| reject(err); | ||
| }); | ||
|
|
||
| worker.on("exit", (code) => { | ||
| if (code !== 0) { | ||
| reject(new Error(`Worker ${i + 1} stopped with exit code ${code}`)); | ||
| } else { | ||
| resolve(); | ||
| } | ||
| }); | ||
| }) | ||
| ); | ||
| } | ||
|
|
||
| try { | ||
| await Promise.all(promises); | ||
| } catch (error) { | ||
| console.error("Error in worker threads: ", error); | ||
| throw error; | ||
| } | ||
| }; | ||
|
|
||
| // Function to execute ramp-ups | ||
| const executeRampUps = async () => { | ||
| let activeThreads = 1; | ||
| let startDoc = 0; | ||
|
|
||
| for (let i = 0; i < rampUps; i++) { | ||
| await spawnWorkers(activeThreads, startDoc, docsPerRampUp); | ||
| startDoc += docsPerRampUp; | ||
|
|
||
| if (activeThreads < maxThreads) { | ||
| activeThreads++; // Increase the number of threads for next ramp-up | ||
| } | ||
|
|
||
| if (i < rampUps - 1) { | ||
| console.log( | ||
| `Ramping up to ${activeThreads} worker(s) in ${ | ||
| rampUpDelay / 1000 | ||
| } seconds...` | ||
| ); | ||
| await new Promise((resolve) => setTimeout(resolve, rampUpDelay)); | ||
| } | ||
| } | ||
| }; | ||
|
|
||
| // Run the ramp-ups | ||
| executeRampUps() | ||
| .then(() => { | ||
| const totalEndTime = performance.now(); | ||
| const totalDuration = (totalEndTime - totalStartTime) / 1000; // Convert to seconds | ||
| console.log( | ||
| `Successfully written ${totalDocs} documents to the collection in ${totalDuration.toFixed( | ||
| 2 | ||
| )} seconds.` | ||
| ); | ||
| }) | ||
| .catch((error) => { | ||
| console.error("Error in worker threads: ", error); | ||
| }); |
Oops, something went wrong.