From b51254cc4969c3975238d833fe740a98508a6de6 Mon Sep 17 00:00:00 2001 From: Manuel Ruck Date: Sun, 22 Oct 2023 11:30:58 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=F0=9F=90=9B=20exit=20job=20correctly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Ruck --- .../import-named-poll-deputies/src/index.ts | 96 ++++++++----------- .../cron-jobs/push-send-queued/src/index.ts | 4 +- .../sync-deputy-profiles/src/index.ts | 11 ++- .../sync-deputy-profiles/src/mongoose.ts | 1 + services/cron-jobs/sync-procedures/env.local | 3 + .../cron-jobs/sync-procedures/src/index.ts | 11 +-- .../cron-jobs/sync-procedures/src/mongoose.ts | 8 -- 7 files changed, 56 insertions(+), 78 deletions(-) create mode 100644 services/cron-jobs/sync-procedures/env.local diff --git a/services/cron-jobs/import-named-poll-deputies/src/index.ts b/services/cron-jobs/import-named-poll-deputies/src/index.ts index 0a51dbdaa..50b2b86f3 100644 --- a/services/cron-jobs/import-named-poll-deputies/src/index.ts +++ b/services/cron-jobs/import-named-poll-deputies/src/index.ts @@ -1,68 +1,50 @@ -import mongoConnect from "./mongoose"; +import mongoConnect from './mongoose'; -import { Scraper } from "@democracy-deutschland/scapacra"; -import { NamedPollDeputyScraper } from "@democracy-deutschland/scapacra-bt"; +import { Scraper } from '@democracy-deutschland/scapacra'; +import { NamedPollDeputyScraper } from '@democracy-deutschland/scapacra-bt'; -import { - NamedPollModel, - setCronStart, - setCronSuccess, - setCronError, -} from "@democracy-deutschland/bundestagio-common"; +import { NamedPollModel, setCronStart, setCronSuccess, setCronError } from '@democracy-deutschland/bundestagio-common'; -const CRON_NAME = "NamedPollsDeputies"; +const CRON_NAME = 'NamedPollsDeputies'; const start = async () => { const startDate = new Date(); await setCronStart({ name: CRON_NAME, startDate }); try { - await Scraper.scrape( - new NamedPollDeputyScraper(), - async (dataPackage: any) => { - process.stdout.write("."); - // Construct Database object - const namedPoll: any = { webId: dataPackage.data.id }; - // Add webId field, Remove id field - const deputies = dataPackage.data.votes.deputies.reduce( - (accumulator: any, deputy: any) => { - // Remove deputies without an id; - if (!deputy.id) { - return accumulator; - } - const dep = deputy; - dep.webId = dep.id; - delete dep.id; - return [...accumulator, dep]; - }, - [] - ); - - const existingNamedPoll = await NamedPollModel.findOne({ - webId: namedPoll.webId, - }); - - // votes.deputies - if ( - !existingNamedPoll || - !existingNamedPoll.votes || - !( - JSON.stringify(existingNamedPoll.votes.deputies) === - JSON.stringify(deputies) - ) - ) { - namedPoll["votes.deputies"] = deputies; + await Scraper.scrape(new NamedPollDeputyScraper(), async (dataPackage: any) => { + console.log('id:', dataPackage.data.id); + // Construct Database object + const namedPoll: any = { webId: dataPackage.data.id }; + // Add webId field, Remove id field + const deputies = dataPackage.data.votes.deputies.reduce((accumulator: any, deputy: any) => { + // Remove deputies without an id; + if (!deputy.id) { + return accumulator; } + const dep = deputy; + dep.webId = dep.id; + delete dep.id; + return [...accumulator, dep]; + }, []); - // Update/Insert - await NamedPollModel.findOneAndUpdate( - { webId: namedPoll.webId }, - { $set: namedPoll }, - { upsert: true } - ); + const existingNamedPoll = await NamedPollModel.findOne({ + webId: namedPoll.webId, + }); - return; + // votes.deputies + if ( + !existingNamedPoll || + !existingNamedPoll.votes || + !(JSON.stringify(existingNamedPoll.votes.deputies) === JSON.stringify(deputies)) + ) { + namedPoll['votes.deputies'] = deputies; } - ); + + // Update/Insert + await NamedPollModel.findOneAndUpdate({ webId: namedPoll.webId }, { $set: namedPoll }, { upsert: true }); + + return; + }); } catch (error) { await setCronError({ name: CRON_NAME, error: JSON.stringify(error) }); throw error; @@ -71,13 +53,13 @@ const start = async () => { }; (async () => { - console.info("START"); - console.info("process.env", process.env.DB_URL); + console.info('START'); + console.info('process.env', process.env.DB_URL); if (!process.env.DB_URL) { - throw new Error("you have to set environment variable: DB_URL"); + throw new Error('you have to set environment variable: DB_URL'); } await mongoConnect(); - console.log("procedures", await NamedPollModel.countDocuments({})); + console.log('procedures', await NamedPollModel.countDocuments({})); await start().catch(() => process.exit(1)); process.exit(0); })(); diff --git a/services/cron-jobs/push-send-queued/src/index.ts b/services/cron-jobs/push-send-queued/src/index.ts index f2b076f9d..397d15eea 100644 --- a/services/cron-jobs/push-send-queued/src/index.ts +++ b/services/cron-jobs/push-send-queued/src/index.ts @@ -97,7 +97,7 @@ const start = async () => { await start().catch((e) => { throw e; }); +})().finally(async () => { + await mongoDisconnect(); process.exit(0); -})().finally(() => { - mongoDisconnect(); }); diff --git a/services/cron-jobs/sync-deputy-profiles/src/index.ts b/services/cron-jobs/sync-deputy-profiles/src/index.ts index 563c36b7f..7e13f9bad 100644 --- a/services/cron-jobs/sync-deputy-profiles/src/index.ts +++ b/services/cron-jobs/sync-deputy-profiles/src/index.ts @@ -120,7 +120,10 @@ const start = async () => { console.log('procedures', await DeputyModel.countDocuments({})); await start(); await mongoDisconnect(); -})().catch(async (e) => { - await mongoDisconnect(); - throw e; -}); +})() + .catch(async () => { + process.exit(1); + }) + .then(async () => { + process.exit(0); + }); diff --git a/services/cron-jobs/sync-deputy-profiles/src/mongoose.ts b/services/cron-jobs/sync-deputy-profiles/src/mongoose.ts index 85050713a..eaeeb3249 100644 --- a/services/cron-jobs/sync-deputy-profiles/src/mongoose.ts +++ b/services/cron-jobs/sync-deputy-profiles/src/mongoose.ts @@ -24,4 +24,5 @@ export const mongoDisconnect = () => { if (connection) { return connection.disconnect(); } + return; }; diff --git a/services/cron-jobs/sync-procedures/env.local b/services/cron-jobs/sync-procedures/env.local new file mode 100644 index 000000000..040509219 --- /dev/null +++ b/services/cron-jobs/sync-procedures/env.local @@ -0,0 +1,3 @@ +BUNDESTAGIO_SERVER_URL=http://localhost:3100 +LIMIT=10 +DB_URL=mongodb://localhost/democracy \ No newline at end of file diff --git a/services/cron-jobs/sync-procedures/src/index.ts b/services/cron-jobs/sync-procedures/src/index.ts index ff496a367..61b367d54 100644 --- a/services/cron-jobs/sync-procedures/src/index.ts +++ b/services/cron-jobs/sync-procedures/src/index.ts @@ -1,4 +1,4 @@ -import { mongoConnect, mongoDisconnect } from './mongoose'; +import { mongoConnect } from './mongoose'; import _ from 'lodash'; import moment from 'moment'; import { forEachSeries } from 'p-iteration'; @@ -322,10 +322,7 @@ const start = async () => { throw new Error('you have to set environment variable: BUNDESTAGIO_SERVER_URL & DB_URL'); } await mongoConnect(); - console.log('procedures', await ProcedureModel.countDocuments({})); await start(); - await mongoDisconnect(); -})().catch(async (e) => { - await mongoDisconnect(); - throw e; -}); + console.info('END'); + process.exit(); +})(); diff --git a/services/cron-jobs/sync-procedures/src/mongoose.ts b/services/cron-jobs/sync-procedures/src/mongoose.ts index 85050713a..c10619ee4 100644 --- a/services/cron-jobs/sync-procedures/src/mongoose.ts +++ b/services/cron-jobs/sync-procedures/src/mongoose.ts @@ -1,7 +1,5 @@ import { mongoose } from '@democracy-deutschland/democracy-common'; -let connection: typeof mongoose; - export const mongoConnect = async () => new Promise(async (resolve) => { // Mongo Debug @@ -19,9 +17,3 @@ export const mongoConnect = async () => throw e; }); }); - -export const mongoDisconnect = () => { - if (connection) { - return connection.disconnect(); - } -};