From ca4f119f966c666528479a18d19bcb735460a387 Mon Sep 17 00:00:00 2001 From: Manuel Ruck Date: Sat, 16 Mar 2024 21:28:22 +0100 Subject: [PATCH] fix: add map missing data Signed-off-by: Manuel Ruck --- pnpm-lock.yaml | 177 ++++++++++++------ services/cron-jobs/crawler/package.json | 3 +- .../import-procedures/import-procedures.ts | 120 ++++++++++-- .../crawler/src/import-procedures/index.ts | 4 +- .../crawler/src/import-procedures/utils.ts | 5 + 5 files changed, 232 insertions(+), 77 deletions(-) create mode 100644 services/cron-jobs/crawler/src/import-procedures/utils.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 881e4e31a..d11d9c01f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -162,7 +162,7 @@ importers: devDependencies: '@graphql-codegen/cli': specifier: 5.0.0 - version: 5.0.0(@types/node@20.10.4)(graphql@16.8.1)(typescript@5.2.2) + version: 5.0.0(@types/node@20.11.28)(graphql@16.8.1)(typescript@5.2.2) '@graphql-codegen/typescript': specifier: 4.0.1 version: 4.0.1(graphql@16.8.1) @@ -189,13 +189,13 @@ importers: version: 5.0.1(eslint-config-prettier@9.0.0)(eslint@8.52.0)(prettier@3.0.3) jest: specifier: 29.7.0 - version: 29.7.0(@types/node@20.10.4) + version: 29.7.0(@types/node@20.11.28) prettier: specifier: 3.0.3 version: 3.0.3 ts-node-dev: specifier: 2.0.0 - version: 2.0.0(@swc/core@1.3.93)(@types/node@20.10.4)(typescript@5.2.2) + version: 2.0.0(@swc/core@1.3.93)(@types/node@20.11.28)(typescript@5.2.2) ts-unused-exports: specifier: 10.0.1 version: 10.0.1(typescript@5.2.2) @@ -319,8 +319,8 @@ importers: services/cron-jobs/crawler: dependencies: '@democracy-deutschland/bt-dip-sdk': - specifier: ^0.1.0-alpha.8 - version: 0.1.1 + specifier: 1.3.0-alpha.0 + version: 1.3.0-alpha.0(axios@1.6.0) '@democracy-deutschland/bundestag.io-definitions': specifier: ^1.0.2 version: 1.0.2 @@ -343,6 +343,9 @@ importers: '@types/jest': specifier: ^26.0.24 version: 26.0.24 + '@types/node': + specifier: ^20.11.28 + version: 20.11.28 '@typescript-eslint/eslint-plugin': specifier: ^6 version: 6.7.2(@typescript-eslint/parser@6.7.2)(eslint@7.32.0)(typescript@5.2.2) @@ -477,7 +480,7 @@ importers: version: 16.3.1 ts-node-dev: specifier: ^2.0.0 - version: 2.0.0(@swc/core@1.3.93)(@types/node@20.10.4)(typescript@5.2.2) + version: 2.0.0(@swc/core@1.3.93)(@types/node@20.11.28)(typescript@5.2.2) ts-unused-exports: specifier: ^10.0.1 version: 10.0.1(typescript@5.2.2) @@ -623,7 +626,7 @@ importers: version: 16.3.1 ts-node-dev: specifier: ^2.0.0 - version: 2.0.0(@swc/core@1.3.93)(@types/node@20.10.4)(typescript@5.2.2) + version: 2.0.0(@swc/core@1.3.93)(@types/node@20.11.28)(typescript@5.2.2) ts-unused-exports: specifier: ^9.0.2 version: 9.0.5(typescript@5.2.2) @@ -928,7 +931,7 @@ importers: version: 5.11.97 ts-node-dev: specifier: ^2.0.0 - version: 2.0.0(@swc/core@1.3.93)(@types/node@20.10.4)(typescript@5.2.2) + version: 2.0.0(@swc/core@1.3.93)(@types/node@20.11.28)(typescript@5.2.2) tsup: specifier: ^7.2.0 version: 7.2.0(@swc/core@1.3.93)(typescript@5.2.2) @@ -2607,8 +2610,12 @@ packages: kuler: 2.0.0 dev: false - /@democracy-deutschland/bt-dip-sdk@0.1.1: - resolution: {integrity: sha512-uH+xswTo9gnGm5F76A6aUxhXvihjVkYpquhI/S7kSAJxRmZ9e5nWNK0tMoOOeHiTVZNGucBq2XSvTuzsB6aoOg==} + /@democracy-deutschland/bt-dip-sdk@1.3.0-alpha.0(axios@1.6.0): + resolution: {integrity: sha512-zg2XhDeBemiEfi+eTD4rUvILFB4nhXSsOBrQeODJLM7OF3bI4xTPflIj3gXflGAs39BfhfzGtcf5Bvnz8UlFfA==} + peerDependencies: + axios: '*' + dependencies: + axios: 1.6.0(debug@4.3.4) dev: false /@democracy-deutschland/bundestag.io-definitions@1.0.2: @@ -3059,7 +3066,7 @@ packages: dev: false optional: true - /@graphql-codegen/cli@5.0.0(@types/node@20.10.4)(graphql@16.8.1)(typescript@5.2.2): + /@graphql-codegen/cli@5.0.0(@types/node@20.11.28)(graphql@16.8.1)(typescript@5.2.2): resolution: {integrity: sha512-A7J7+be/a6e+/ul2KI5sfJlpoqeqwX8EzktaKCeduyVKgOLA6W5t+NUGf6QumBDXU8PEOqXk3o3F+RAwCWOiqA==} hasBin: true peerDependencies: @@ -3077,12 +3084,12 @@ packages: '@graphql-tools/apollo-engine-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/code-file-loader': 8.0.2(graphql@16.8.1) '@graphql-tools/git-loader': 8.0.2(graphql@16.8.1) - '@graphql-tools/github-loader': 8.0.0(@types/node@20.10.4)(graphql@16.8.1) + '@graphql-tools/github-loader': 8.0.0(@types/node@20.11.28)(graphql@16.8.1) '@graphql-tools/graphql-file-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/json-file-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/load': 8.0.0(graphql@16.8.1) - '@graphql-tools/prisma-loader': 8.0.1(@types/node@20.10.4)(graphql@16.8.1) - '@graphql-tools/url-loader': 8.0.0(@types/node@20.10.4)(graphql@16.8.1) + '@graphql-tools/prisma-loader': 8.0.1(@types/node@20.11.28)(graphql@16.8.1) + '@graphql-tools/url-loader': 8.0.0(@types/node@20.11.28)(graphql@16.8.1) '@graphql-tools/utils': 10.0.8(graphql@16.8.1) '@whatwg-node/fetch': 0.8.8 chalk: 4.1.2 @@ -3090,7 +3097,7 @@ packages: debounce: 1.2.1 detect-indent: 6.1.0 graphql: 16.8.1 - graphql-config: 5.0.3(@types/node@20.10.4)(graphql@16.8.1)(typescript@5.2.2) + graphql-config: 5.0.3(@types/node@20.11.28)(graphql@16.8.1)(typescript@5.2.2) inquirer: 8.2.6 is-glob: 4.0.3 jiti: 1.20.0 @@ -3299,7 +3306,7 @@ packages: - utf-8-validate dev: true - /@graphql-tools/executor-http@1.0.3(@types/node@20.10.4)(graphql@16.8.1): + /@graphql-tools/executor-http@1.0.3(@types/node@20.11.28)(graphql@16.8.1): resolution: {integrity: sha512-5WZIMBevRaxMabZ8U2Ty0dTUPy/PpeYSlMNEmC/YJjKKykgSfc/AwSejx2sE4FFKZ0I2kxRKRenyoWMHRAV49Q==} engines: {node: '>=16.0.0'} peerDependencies: @@ -3310,7 +3317,7 @@ packages: '@whatwg-node/fetch': 0.9.13 extract-files: 11.0.0 graphql: 16.8.1 - meros: 1.3.0(@types/node@20.10.4) + meros: 1.3.0(@types/node@20.11.28) tslib: 2.6.2 value-or-promise: 1.0.12 transitivePeerDependencies: @@ -3365,14 +3372,14 @@ packages: - supports-color dev: true - /@graphql-tools/github-loader@8.0.0(@types/node@20.10.4)(graphql@16.8.1): + /@graphql-tools/github-loader@8.0.0(@types/node@20.11.28)(graphql@16.8.1): resolution: {integrity: sha512-VuroArWKcG4yaOWzV0r19ElVIV6iH6UKDQn1MXemND0xu5TzrFme0kf3U9o0YwNo0kUYEk9CyFM0BYg4he17FA==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: '@ardatan/sync-fetch': 0.0.1 - '@graphql-tools/executor-http': 1.0.3(@types/node@20.10.4)(graphql@16.8.1) + '@graphql-tools/executor-http': 1.0.3(@types/node@20.11.28)(graphql@16.8.1) '@graphql-tools/graphql-tag-pluck': 8.0.2(graphql@16.8.1) '@graphql-tools/utils': 10.0.8(graphql@16.8.1) '@whatwg-node/fetch': 0.9.13 @@ -3485,13 +3492,13 @@ packages: tslib: 2.5.3 dev: true - /@graphql-tools/prisma-loader@8.0.1(@types/node@20.10.4)(graphql@16.8.1): + /@graphql-tools/prisma-loader@8.0.1(@types/node@20.11.28)(graphql@16.8.1): resolution: {integrity: sha512-bl6e5sAYe35Z6fEbgKXNrqRhXlCJYeWKBkarohgYA338/SD9eEhXtg3Cedj7fut3WyRLoQFpHzfiwxKs7XrgXg==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - '@graphql-tools/url-loader': 8.0.0(@types/node@20.10.4)(graphql@16.8.1) + '@graphql-tools/url-loader': 8.0.0(@types/node@20.11.28)(graphql@16.8.1) '@graphql-tools/utils': 10.0.8(graphql@16.8.1) '@types/js-yaml': 4.0.8 '@types/json-stable-stringify': 1.0.35 @@ -3557,7 +3564,7 @@ packages: value-or-promise: 1.0.12 dev: false - /@graphql-tools/url-loader@8.0.0(@types/node@20.10.4)(graphql@16.8.1): + /@graphql-tools/url-loader@8.0.0(@types/node@20.11.28)(graphql@16.8.1): resolution: {integrity: sha512-rPc9oDzMnycvz+X+wrN3PLrhMBQkG4+sd8EzaFN6dypcssiefgWKToXtRKI8HHK68n2xEq1PyrOpkjHFJB+GwA==} engines: {node: '>=16.0.0'} peerDependencies: @@ -3566,7 +3573,7 @@ packages: '@ardatan/sync-fetch': 0.0.1 '@graphql-tools/delegate': 10.0.3(graphql@16.8.1) '@graphql-tools/executor-graphql-ws': 1.1.0(graphql@16.8.1) - '@graphql-tools/executor-http': 1.0.3(@types/node@20.10.4)(graphql@16.8.1) + '@graphql-tools/executor-http': 1.0.3(@types/node@20.11.28)(graphql@16.8.1) '@graphql-tools/executor-legacy-ws': 1.0.4(graphql@16.8.1) '@graphql-tools/utils': 10.0.8(graphql@16.8.1) '@graphql-tools/wrap': 10.0.1(graphql@16.8.1) @@ -3706,7 +3713,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 chalk: 4.1.2 jest-message-util: 27.5.1 jest-util: 27.5.1 @@ -3739,7 +3746,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.8.1 @@ -3819,7 +3826,7 @@ packages: dependencies: '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 jest-mock: 27.5.1 dev: true @@ -3856,7 +3863,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@sinonjs/fake-timers': 8.1.0 - '@types/node': 20.10.4 + '@types/node': 20.11.28 jest-message-util: 27.5.1 jest-mock: 27.5.1 jest-util: 27.5.1 @@ -3909,7 +3916,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -4089,7 +4096,7 @@ packages: dependencies: '@types/istanbul-lib-coverage': 2.0.4 '@types/istanbul-reports': 3.0.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 '@types/yargs': 15.0.15 chalk: 4.1.2 dev: true @@ -4100,7 +4107,7 @@ packages: dependencies: '@types/istanbul-lib-coverage': 2.0.4 '@types/istanbul-reports': 3.0.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 '@types/yargs': 16.0.5 chalk: 4.1.2 dev: true @@ -5103,7 +5110,7 @@ packages: /@types/graceful-fs@4.1.8: resolution: {integrity: sha512-NhRH7YzWq8WiNKVavKPBmtLYZHxNY19Hh+az28O/phfp68CF45pMFud+ZzJ8ewnxnC5smIdF3dqFeiSUQ5I+pw==} dependencies: - '@types/node': 20.10.4 + '@types/node': 20.11.28 dev: true /@types/graphql-date@1.0.8: @@ -5292,13 +5299,21 @@ packages: dependencies: undici-types: 5.26.5 + /@types/node@20.11.28: + resolution: {integrity: sha512-M/GPWVS2wLkSkNHVeLkrF2fD5Lx5UC4PxA0uZcKc6QqbIQUJyW1jVjueJYi1z8n0I5PxYrtpnPnWglE+y9A0KA==} + dependencies: + undici-types: 5.26.5 + dev: true + /@types/node@20.6.4: resolution: {integrity: sha512-nU6d9MPY0NBUMiE/nXd2IIoC4OLvsLpwAjheoAeuzgvDZA1Cb10QYg+91AF6zQiKWRN5i1m07x6sMe0niBznoQ==} + dev: true /@types/node@20.8.10: resolution: {integrity: sha512-TlgT8JntpcbmKUFzjhsyhGfP2fsiz1Mv56im6enJ905xG1DAYesxJaeSbGqQmAw8OWPdhyJGhGSQGKRNJ45u9w==} dependencies: undici-types: 5.26.5 + dev: true /@types/node@20.8.7: resolution: {integrity: sha512-21TKHHh3eUHIi2MloeptJWALuCu5H7HQTdTrWIFReA8ad+aggoX+lRes3ex7/FtpC+sVUpFMQ+QTfYr74mruiQ==} @@ -5346,7 +5361,7 @@ packages: resolution: {integrity: sha512-HuihY1+Vss5RS9ZHzRyTGIzwPTdrJBkCm/mAeLRYrOQu/MGqyezKXWOK1VhCnR+SDbp9G2mRUP+OVEqCrzpcfA==} dependencies: '@types/caseless': 0.12.4 - '@types/node': 20.8.10 + '@types/node': 20.10.4 '@types/tough-cookie': 4.0.3 form-data: 2.5.1 dev: true @@ -5451,7 +5466,7 @@ packages: /@types/xml2js@0.4.13: resolution: {integrity: sha512-nuT42GzgoUa4zZgBoF4d+Zqc12/FlVxXCT4xU6j3RfqTFVQWrUAClI/0sNJ5ImM9Wv6KB42KMG2xsVMn4cSBzA==} dependencies: - '@types/node': 20.8.10 + '@types/node': 20.10.4 dev: false /@types/xmldom@0.1.33: @@ -5461,7 +5476,7 @@ packages: /@types/xsd-schema-validator@0.5.6: resolution: {integrity: sha512-ZAnzytBMn7CZVqAkaesPscbGQL22+S+7QTwtQKRtmVuSGxkj116xFQzi8Bc8CengQ6wZcGvuJ3wiVm21+dFeBw==} dependencies: - '@types/node': 20.8.10 + '@types/node': 20.10.4 dev: true /@types/yargs-parser@21.0.0: @@ -5980,7 +5995,7 @@ packages: /@wry/context@0.4.4: resolution: {integrity: sha512-LrKVLove/zw6h2Md/KZyWxIkFM6AoyKp71OqpH9Hiip1csjPVoD3tPxlbQUNxEnHENks3UGgNpSBCAfq9KWuag==} dependencies: - '@types/node': 20.6.4 + '@types/node': 20.10.4 tslib: 1.14.1 dev: false @@ -7959,7 +7974,7 @@ packages: - utf-8-validate dev: false - /create-jest@29.7.0(@types/node@20.10.4): + /create-jest@29.7.0(@types/node@20.11.28): resolution: {integrity: sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} hasBin: true @@ -7968,7 +7983,7 @@ packages: chalk: 4.1.2 exit: 0.1.2 graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@20.10.4) + jest-config: 29.7.0(@types/node@20.11.28) jest-util: 29.7.0 prompts: 2.4.2 transitivePeerDependencies: @@ -10389,7 +10404,7 @@ packages: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} dev: true - /graphql-config@5.0.3(@types/node@20.10.4)(graphql@16.8.1)(typescript@5.2.2): + /graphql-config@5.0.3(@types/node@20.11.28)(graphql@16.8.1)(typescript@5.2.2): resolution: {integrity: sha512-BNGZaoxIBkv9yy6Y7omvsaBUHOzfFcII3UN++tpH8MGOKFPFkCPZuwx09ggANMt8FgyWP1Od8SWPmrUEZca4NQ==} engines: {node: '>= 16.0.0'} peerDependencies: @@ -10403,7 +10418,7 @@ packages: '@graphql-tools/json-file-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/load': 8.0.0(graphql@16.8.1) '@graphql-tools/merge': 9.0.0(graphql@16.8.1) - '@graphql-tools/url-loader': 8.0.0(@types/node@20.10.4)(graphql@16.8.1) + '@graphql-tools/url-loader': 8.0.0(@types/node@20.11.28)(graphql@16.8.1) '@graphql-tools/utils': 10.0.8(graphql@16.8.1) cosmiconfig: 8.3.6(typescript@5.2.2) graphql: 16.8.1 @@ -11436,7 +11451,7 @@ packages: '@jest/environment': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 chalk: 4.1.2 co: 4.6.0 dedent: 0.7.0 @@ -11515,7 +11530,7 @@ packages: - utf-8-validate dev: true - /jest-cli@29.7.0(@types/node@20.10.4): + /jest-cli@29.7.0(@types/node@20.11.28): resolution: {integrity: sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} hasBin: true @@ -11529,10 +11544,10 @@ packages: '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 chalk: 4.1.2 - create-jest: 29.7.0(@types/node@20.10.4) + create-jest: 29.7.0(@types/node@20.11.28) exit: 0.1.2 import-local: 3.1.0 - jest-config: 29.7.0(@types/node@20.10.4) + jest-config: 29.7.0(@types/node@20.11.28) jest-util: 29.7.0 jest-validate: 29.7.0 yargs: 17.7.2 @@ -11623,6 +11638,46 @@ packages: - supports-color dev: true + /jest-config@29.7.0(@types/node@20.11.28): + resolution: {integrity: sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + peerDependencies: + '@types/node': '*' + ts-node: '>=9.0.0' + peerDependenciesMeta: + '@types/node': + optional: true + ts-node: + optional: true + dependencies: + '@babel/core': 7.23.2 + '@jest/test-sequencer': 29.7.0 + '@jest/types': 29.6.3 + '@types/node': 20.11.28 + babel-jest: 29.7.0(@babel/core@7.23.2) + chalk: 4.1.2 + ci-info: 3.9.0 + deepmerge: 4.3.1 + glob: 7.2.3 + graceful-fs: 4.2.11 + jest-circus: 29.7.0 + jest-environment-node: 29.7.0 + jest-get-type: 29.6.3 + jest-regex-util: 29.6.3 + jest-resolve: 29.7.0 + jest-runner: 29.7.0 + jest-util: 29.7.0 + jest-validate: 29.7.0 + micromatch: 4.0.5 + parse-json: 5.2.0 + pretty-format: 29.7.0 + slash: 3.0.0 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + dev: true + /jest-diff@26.6.2: resolution: {integrity: sha512-6m+9Z3Gv9wN0WFVasqjCL/06+EFCMTqDEUl/b87HYK2rAPTyfz4ZIuSlPhY51PIQRWx5TaxeF1qmXKe9gfN3sA==} engines: {node: '>= 10.14.2'} @@ -11696,7 +11751,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 jest-mock: 27.5.1 jest-util: 27.5.1 jsdom: 16.7.0 @@ -11714,7 +11769,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 jest-mock: 27.5.1 jest-util: 27.5.1 dev: true @@ -11752,7 +11807,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@types/graceful-fs': 4.1.8 - '@types/node': 20.10.4 + '@types/node': 20.11.28 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -11793,7 +11848,7 @@ packages: '@jest/source-map': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 chalk: 4.1.2 co: 4.6.0 expect: 27.5.1 @@ -11881,7 +11936,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 dev: true /jest-mock@29.7.0: @@ -11988,7 +12043,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 chalk: 4.1.2 emittery: 0.8.1 graceful-fs: 4.2.11 @@ -12104,7 +12159,7 @@ packages: resolution: {integrity: sha512-jZCyo6iIxO1aqUxpuBlwTDMkzOAJS4a3eYz3YzgxxVQFwLeSA7Jfq5cbqCY+JLvTDrWirgusI/0KwxKMgrdf7w==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@types/node': 20.10.4 + '@types/node': 20.11.28 graceful-fs: 4.2.11 dev: true @@ -12171,7 +12226,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 chalk: 4.1.2 ci-info: 3.8.0 graceful-fs: 4.2.11 @@ -12220,7 +12275,7 @@ packages: dependencies: '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.10.4 + '@types/node': 20.11.28 ansi-escapes: 4.3.2 chalk: 4.1.2 jest-util: 27.5.1 @@ -12245,7 +12300,7 @@ packages: resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} engines: {node: '>= 10.13.0'} dependencies: - '@types/node': 20.10.4 + '@types/node': 20.11.28 merge-stream: 2.0.0 supports-color: 8.1.1 dev: true @@ -12281,7 +12336,7 @@ packages: - utf-8-validate dev: true - /jest@29.7.0(@types/node@20.10.4): + /jest@29.7.0(@types/node@20.11.28): resolution: {integrity: sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} hasBin: true @@ -12294,7 +12349,7 @@ packages: '@jest/core': 29.7.0 '@jest/types': 29.6.3 import-local: 3.1.0 - jest-cli: 29.7.0(@types/node@20.10.4) + jest-cli: 29.7.0(@types/node@20.11.28) transitivePeerDependencies: - '@types/node' - babel-plugin-macros @@ -13196,7 +13251,7 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} - /meros@1.3.0(@types/node@20.10.4): + /meros@1.3.0(@types/node@20.11.28): resolution: {integrity: sha512-2BNGOimxEz5hmjUG2FwoxCt5HN7BXdaWyFqEwxPTrJzVdABtrL4TiHTcsWSFAxPQ/tOnEaQEJh3qWq71QRMY+w==} engines: {node: '>=13'} peerDependencies: @@ -13205,7 +13260,7 @@ packages: '@types/node': optional: true dependencies: - '@types/node': 20.10.4 + '@types/node': 20.11.28 dev: true /methods@1.1.2: @@ -16629,7 +16684,7 @@ packages: typescript: 5.2.2 dev: true - /ts-node-dev@2.0.0(@swc/core@1.3.93)(@types/node@20.10.4)(typescript@5.2.2): + /ts-node-dev@2.0.0(@swc/core@1.3.93)(@types/node@20.11.28)(typescript@5.2.2): resolution: {integrity: sha512-ywMrhCfH6M75yftYvrvNarLEY+SUXtUvU8/0Z6llrHQVBx12GiFk5sStF8UdfE/yfzk9IAq7O5EEbTQsxlBI8w==} engines: {node: '>=0.8.0'} hasBin: true @@ -16648,7 +16703,7 @@ packages: rimraf: 2.7.1 source-map-support: 0.5.21 tree-kill: 1.2.2 - ts-node: 10.9.1(@swc/core@1.3.93)(@types/node@20.10.4)(typescript@5.2.2) + ts-node: 10.9.1(@swc/core@1.3.93)(@types/node@20.11.28)(typescript@5.2.2) tsconfig: 7.0.0 typescript: 5.2.2 transitivePeerDependencies: @@ -16713,7 +16768,7 @@ packages: - '@types/node' dev: true - /ts-node@10.9.1(@swc/core@1.3.93)(@types/node@20.10.4)(typescript@5.2.2): + /ts-node@10.9.1(@swc/core@1.3.93)(@types/node@20.11.28)(typescript@5.2.2): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -16733,7 +16788,7 @@ packages: '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.10.4 + '@types/node': 20.11.28 acorn: 8.11.2 acorn-walk: 8.3.0 arg: 4.1.3 diff --git a/services/cron-jobs/crawler/package.json b/services/cron-jobs/crawler/package.json index 137b164fb..f860e417e 100644 --- a/services/cron-jobs/crawler/package.json +++ b/services/cron-jobs/crawler/package.json @@ -14,7 +14,7 @@ "start": "node ./build/index.js" }, "dependencies": { - "@democracy-deutschland/bt-dip-sdk": "^0.1.0-alpha.8", + "@democracy-deutschland/bt-dip-sdk": "1.3.0-alpha.0", "@democracy-deutschland/bundestag.io-definitions": "^1.0.2", "@democracy-deutschland/bundestagio-common": "workspace:*", "axios": "^1.6.0", @@ -24,6 +24,7 @@ "devDependencies": { "@types/debug": "^4.1.5", "@types/jest": "^26.0.24", + "@types/node": "^20.11.28", "@typescript-eslint/eslint-plugin": "^6", "@typescript-eslint/parser": "^6", "eslint": "^7.31.0", diff --git a/services/cron-jobs/crawler/src/import-procedures/import-procedures.ts b/services/cron-jobs/crawler/src/import-procedures/import-procedures.ts index 2486a2407..a897da48a 100644 --- a/services/cron-jobs/crawler/src/import-procedures/import-procedures.ts +++ b/services/cron-jobs/crawler/src/import-procedures/import-procedures.ts @@ -1,15 +1,26 @@ import { ProcedureModel } from '@democracy-deutschland/bundestagio-common'; import debug from 'debug'; import { CONFIG } from '../config'; -import { Configuration, DipApi, Vorgang } from '@democracy-deutschland/bt-dip-sdk'; +import { + Configuration, + Vorgang, + VorgngeApi, + VorgangspositionenApi, + Vorgangsposition, +} from '@democracy-deutschland/bt-dip-sdk'; const log = debug('bundestag-io:import-procedures:log'); log.log = console.log.bind(console); import axios from '../axios'; +import { IProcedure } from '@democracy-deutschland/bundestagio-common/dist/models/Procedure/schema'; +import { IDocument } from '@democracy-deutschland/bundestagio-common/dist/models/Procedure/Procedure/Document'; +import { IProcessFlow } from '@democracy-deutschland/bundestagio-common/dist/models/Procedure/Procedure/ProcessFlow'; +import { germanDateFormat } from './utils'; const config = new Configuration({ apiKey: `ApiKey ${CONFIG.DIP_API_KEY}`, // Replace #YOUR_API_KEY# with your api key }); -const dipAPI = new DipApi(config, undefined, axios); +const vorgangApi = new VorgngeApi(config, undefined, axios); +const vorgangspositionenApi = new VorgangspositionenApi(config, undefined, axios); export default async function importProcedures(config: typeof CONFIG): Promise { const { @@ -53,15 +64,93 @@ export default async function importProcedures(config: typeof CONFIG): Promise ({ ...node, procedureId: node.id })); + let procedures: Partial[] = []; + + for (const edge of edges) { + log(`${edge.node.aktualisiert} ${edge.node.id} - ${edge.node.titel}`); + const { node } = edge; + const vorgangspositionen = await vorgangspositionenApi + .getVorgangspositionList({ + fVorgang: Number(node.id), + }) + .then((res) => res.data.documents); + + const importantDocuments = vorgangspositionen + .filter((v) => v.fundstelle?.dokumentart === 'Drucksache') + .map((d) => ({ + editor: d.fundstelle.herausgeber, + number: d.fundstelle.dokumentnummer, + type: d.fundstelle.drucksachetyp!, + url: d.fundstelle.pdf_url!, + })); + + const legalValidity = + node.inkrafttreten?.map((i) => `${i.datum}${i.erlaeuterung ? ` (${i.erlaeuterung})` : ''}`) || []; + + const history = vorgangspositionen.map((d) => { + const getFindSpot = (d: Vorgangsposition) => { + const { fundstelle } = d; + if (!fundstelle) return; + const { herausgeber, dokumentart, dokumentnummer } = fundstelle; + const datum = germanDateFormat.format(new Date(fundstelle.datum)); + const result = `${datum} - ${herausgeber}-${dokumentart} ${dokumentnummer}`; + const { anfangsseite, endseite, anfangsquadrant, endquadrant } = fundstelle; + if (![anfangsseite, endseite, anfangsquadrant, endquadrant].every(Boolean)) return result; + return `${result}, S. ${anfangsseite}${anfangsquadrant} - ${endseite}${endquadrant}`; + }; + + const getDecision = (d: Vorgangsposition) => { + const { beschlussfassung } = d; + if (!beschlussfassung) return []; + return beschlussfassung.map((b) => ({ + page: b.seite, + tenor: b.beschlusstenor, + document: b.dokumentnummer, + type: b.abstimmungsart, + comment: b.abstimm_ergebnis_bemerkung, + foundation: b.grundlage, + majority: b.mehrheit, + })); + }; + + return { + assignment: d.fundstelle.herausgeber, + initiator: + d.fundstelle.urheber.length > 0 + ? `${d.vorgangsposition}, Urheber : ${d.fundstelle.urheber.join(', ')}` + : d.vorgangsposition, + findSpot: getFindSpot(d), + findSpotUrl: d.fundstelle.pdf_url, + decision: getDecision(d), + abstract: d.abstract, + date: new Date(d.fundstelle.datum), + }; + }); + + const procedure: Partial = { + ...node, + procedureId: node.id, + type: node.vorgangstyp, + tags: node.deskriptor?.map((d) => d.name) || [], + title: node.titel, + currentStatus: node.beratungsstand, + period: node.wahlperiode, + subjectGroups: node.sachgebiet, + importantDocuments, + gestOrderNumber: node.gesta, + legalValidity, + history, + }; + + procedures = [...procedures, procedure]; + } await ProcedureModel.bulkWrite( procedures.map((node) => ({ @@ -80,8 +169,8 @@ export default async function importProcedures(config: typeof CONFIG): Promise { - const { before, after, types: typesFilter } = args.filter || {}; + const { start, end, types: typesFilter } = args.filter || {}; const filter: { 'f.datum.start'?: Date; 'f.datum.end'?: Date } = {}; - if (after) filter['f.datum.start'] = after; - if (before) filter['f.datum.end'] = before; + if (start) filter['f.datum.start'] = start; + if (end) filter['f.datum.end'] = end; let hasNextPage = true; let totalCount = 0; let cursor = args.cursor; let documents: Vorgang[] = []; while (documents.length < args.limit + args.offset) { - const { data } = await dipAPI.getVorgaenge({ + console.log({ + cursor, + fAktualisiertStart: args.filter?.start?.toISOString(), + fAktualisiertEnd: args.filter?.end?.toISOString(), + }); + const { data } = await vorgangApi.getVorgangList({ cursor, - fDatumStart: args.filter?.before?.toISOString().slice(0, 10), - fDatumEnd: args.filter?.after?.toISOString().slice(0, 10), + fAktualisiertStart: args.filter?.start?.toISOString(), + fAktualisiertEnd: args.filter?.end?.toISOString(), }); // const res = await this.get(`/api/v1/vorgang`, { ...filter, cursor }); totalCount = Number(data.numFound); diff --git a/services/cron-jobs/crawler/src/import-procedures/index.ts b/services/cron-jobs/crawler/src/import-procedures/index.ts index 3d991351d..9ff6a73cd 100644 --- a/services/cron-jobs/crawler/src/import-procedures/index.ts +++ b/services/cron-jobs/crawler/src/import-procedures/index.ts @@ -5,12 +5,12 @@ import importProcedures from './import-procedures'; (async () => { await mongoConnect(); const cronjob = await getCron({ name: 'import-procedures' }); + console.log('cronjob', cronjob.lastSuccessStartDate); await setCronStart({ name: 'import-procedures' }); await importProcedures({ ...CONFIG, IMPORT_PROCEDURES_FILTER_AFTER: - // cronjob?.lastSuccessDate?.toISOString().slice(0, 10) || - CONFIG.IMPORT_PROCEDURES_FILTER_AFTER, + cronjob?.lastSuccessStartDate?.toISOString() || CONFIG.IMPORT_PROCEDURES_FILTER_AFTER, }); await setCronSuccess({ name: 'import-procedures', successStartDate: cronjob.lastStartDate || new Date() }); process.exit(0); diff --git a/services/cron-jobs/crawler/src/import-procedures/utils.ts b/services/cron-jobs/crawler/src/import-procedures/utils.ts new file mode 100644 index 000000000..fa94f65e9 --- /dev/null +++ b/services/cron-jobs/crawler/src/import-procedures/utils.ts @@ -0,0 +1,5 @@ +export const germanDateFormat = new Intl.DateTimeFormat('de-DE', { + year: 'numeric', + month: '2-digit', + day: '2-digit', +});