From 09704dd1faf4c4baed2182a2dad546f639294e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?De=20Souza=2C=20Manass=C3=A9s?= Date: Sat, 20 Apr 2024 21:31:07 -0300 Subject: [PATCH] [#160] add pubsub implementation for uncategorized --- file_loader.js | 24 +++++++- package-lock.json | 149 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 173 insertions(+), 1 deletion(-) diff --git a/file_loader.js b/file_loader.js index 0ffb10e..fffecf6 100644 --- a/file_loader.js +++ b/file_loader.js @@ -8,15 +8,18 @@ const Map = require("collections/map"); const Dict = require("collections/dict"); const List = require("collections/list"); const Firestore = require('@google-cloud/firestore'); +const {PubSub} = require('@google-cloud/pubsub') const BluebirdPromise = require('bluebird'); const spreadsheet = BluebirdPromise.promisifyAll(require('./spreadsheet')); +const projectId = 'smartfinance-bills-beta' const db = new Firestore({ - projectId: 'smartfinance-bills-beta', + projectId: projectId, keyFilename: process.env.credentials, }); const billsCategoryMap = db.collection('bills_config').doc('mapping') +const pubSubClient = new PubSub(projectId) const FILENAME_DATA_SEPARATOR = "_" const FILENAME_FIELDS_LENGTH = 3 @@ -113,8 +116,14 @@ var self = module.exports = { } } else { console.log('[WARN] category not found: %s', receiptName) + publishMessage('sfinbills', receiptName).catch(err => { + console.error(err.message); + process.exitCode = 1; + }) + return; } }) + console.log("TODO: should stop here and not call spreadsheet") spreadsheet.updateSpreadsheetAsync(spreadsheetMap) .then((result) => { for (let i = 0; i < files.length; i++) { @@ -152,6 +161,19 @@ var self = module.exports = { } }; +async function publishMessage(topic, data) { + const dataBuffer = Buffer.from(data) + try { + const messageId = await pubSubClient + .topic(topic) + .publishMessage({data: dataBuffer}); + console.log(`[INFO] message ${messageId} published.`); + } catch (error) { + console.error(`Received error while publishing: ${error.message}`); + process.exitCode = 1; + } +} + async function fileAlreadyProcessed(file) { let alreadyProcessed = false const querySnapshot = await db.collection('bills').get() diff --git a/package-lock.json b/package-lock.json index a8afa69..b287d63 100644 --- a/package-lock.json +++ b/package-lock.json @@ -58,6 +58,122 @@ "protobufjs": "^7.0.0" } }, + "@google-cloud/paginator": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz", + "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==", + "requires": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + } + }, + "@google-cloud/precise-date": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-3.0.1.tgz", + "integrity": "sha512-crK2rgNFfvLoSgcKJY7ZBOLW91IimVNmPfi1CL+kMTf78pTJYd29XqEVedAeBu4DwCJc0EDIp1MpctLgoPq+Uw==" + }, + "@google-cloud/projectify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz", + "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==" + }, + "@google-cloud/promisify": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-2.0.4.tgz", + "integrity": "sha512-j8yRSSqswWi1QqUGKVEKOG03Q7qOoZP6/h2zN2YO+F5h2+DHU0bSrHCK9Y7lo2DI9fBd8qGAw795sf+3Jva4yA==" + }, + "@google-cloud/pubsub": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/@google-cloud/pubsub/-/pubsub-3.7.3.tgz", + "integrity": "sha512-ZRDC4g7tpIJ8fkAp4MiU+tDfousM/q6pXK6ytFn0cbYEdNQuWOf4wqopNYMOUJ+AIjaTbgmNw77dStOKTc9Acg==", + "requires": { + "@google-cloud/paginator": "^4.0.0", + "@google-cloud/precise-date": "^3.0.0", + "@google-cloud/projectify": "^3.0.0", + "@google-cloud/promisify": "^2.0.0", + "@opentelemetry/api": "^1.0.0", + "@opentelemetry/semantic-conventions": "~1.3.0", + "@types/duplexify": "^3.6.0", + "@types/long": "^4.0.0", + "arrify": "^2.0.0", + "extend": "^3.0.2", + "google-auth-library": "^8.0.2", + "google-gax": "^3.6.1", + "heap-js": "^2.2.0", + "is-stream-ended": "^0.1.4", + "lodash.snakecase": "^4.1.1", + "p-defer": "^3.0.0" + }, + "dependencies": { + "gaxios": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz", + "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==", + "requires": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + } + }, + "gcp-metadata": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz", + "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==", + "requires": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + } + }, + "google-auth-library": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.9.0.tgz", + "integrity": "sha512-f7aQCJODJFmYWN6PeNKzgvy9LI2tYmXnzpNDHEjG5sDNPgGb2FXQyTBnXeSH+PAtpKESFD+LmHw3Ox3mN7e1Fg==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.3.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + } + }, + "google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "requires": { + "node-forge": "^1.3.1" + } + }, + "gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", + "requires": { + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" + } + }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==" + } + } + }, "@grpc/grpc-js": { "version": "1.8.21", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.21.tgz", @@ -166,6 +282,16 @@ "fastq": "^1.6.0" } }, + "@opentelemetry/api": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.8.0.tgz", + "integrity": "sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==" + }, + "@opentelemetry/semantic-conventions": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.3.1.tgz", + "integrity": "sha512-wU5J8rUoo32oSef/rFpOT1HIjLjAv3qIDHkw1QIhODV3OpAVHi5oVzlouozg9obUmZKtbZ0qUe/m7FP0y0yBzA==" + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -270,6 +396,14 @@ "tslib": "^1.9.3" } }, + "@types/duplexify": { + "version": "3.6.4", + "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.4.tgz", + "integrity": "sha512-2eahVPsd+dy3CL6FugAzJcxoraWhUghZGEQJns1kTKfCXWKJ5iG/VkaB05wRVrDKHfOFKqb0X0kXh91eE99RZg==", + "requires": { + "@types/node": "*" + } + }, "@types/glob": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-8.1.0.tgz", @@ -1393,6 +1527,11 @@ "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, + "heap-js": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/heap-js/-/heap-js-2.5.0.tgz", + "integrity": "sha512-kUGoI3p7u6B41z/dp33G6OaL7J4DRqRYwVmeIlwLClx7yaaAy7hoDExnuejTKtuDwfcatGmddHDEOjf6EyIxtQ==" + }, "https-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", @@ -1668,6 +1807,11 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "lodash.snakecase": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", + "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==" + }, "log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -2005,6 +2149,11 @@ "word-wrap": "^1.2.3" } }, + "p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==" + }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", diff --git a/package.json b/package.json index 067f2b8..8848e1a 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "node": "12.22.9" }, "dependencies": { + "@google-cloud/pubsub": "3.7.3", "@google-cloud/firestore": "6.7.0", "async": "2.6.4", "bluebird": "3.7.2",