diff --git a/.dockerignore b/.dockerignore index aeac7b641..7d4e8ba3c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,4 +6,6 @@ docker_volumes .turbo .github build -dist \ No newline at end of file +dist +manifests +.garden \ No newline at end of file diff --git a/.gardenignore b/.gardenignore new file mode 100644 index 000000000..549a7480f --- /dev/null +++ b/.gardenignore @@ -0,0 +1,16 @@ +*.log +/data/ +.vs +node_modules +.github/workflows/*.env +infra/*.yaml +configuration/* +!configuration/*.example +docker_volumes/*/* +!/**/.gitkeep +/services/cron-jobs/push-send-queued/.env +.turbo +out +build +/.env +manifests \ No newline at end of file diff --git a/.gitignore b/.gitignore index 72ab009a1..a0f7e12ed 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ docker_volumes/*/* .turbo out build -/.env \ No newline at end of file +/.env +.garden \ No newline at end of file diff --git a/bundestag.io/admin/.dockerignore b/bundestag.io/admin/.dockerignore index fca9db226..33c145982 100644 --- a/bundestag.io/admin/.dockerignore +++ b/bundestag.io/admin/.dockerignore @@ -4,4 +4,6 @@ logs .env* .next *.log -.turbo \ No newline at end of file +.turbo +manifests +.garden \ No newline at end of file diff --git a/bundestag.io/admin/garden.yml b/bundestag.io/admin/garden.yml new file mode 100644 index 000000000..b10d3393f --- /dev/null +++ b/bundestag.io/admin/garden.yml @@ -0,0 +1,41 @@ +kind: Build +name: admin +description: Admin +type: container +source: + path: ../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: bundestag.io-admin + SERVICE_PATH: bundestag.io/admin + +--- +kind: Deploy +name: admin +type: kubernetes +description: Deploy the Admin +dependencies: [build.admin, deploy.bundestag-io-api] + +spec: + files: [./manifests/*] # <--- Tell Garden what manifests to use + + defaultTarget: # <--- This tells Garden what "target" to use for logs, code syncing and more + kind: Deployment + name: admin + + # Patch the K8s manifests for the api service so that we can set the correct image + patchResources: + - name: admin + kind: Deployment + patch: + spec: + template: + spec: + containers: + - name: admin + image: ${actions.build.admin.outputs.deploymentImageId} # <--- Reference the output from the Build action diff --git a/bundestag.io/admin/manifests/deployment.yaml b/bundestag.io/admin/manifests/deployment.yaml new file mode 100644 index 000000000..f436f0143 --- /dev/null +++ b/bundestag.io/admin/manifests/deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin + labels: + app: admin +spec: + replicas: 1 + selector: + matchLabels: + app: admin + template: + metadata: + labels: + app: admin + spec: + containers: + - name: admin + image: democracy/bundestag.io-admin:3.2.0 + imagePullPolicy: IfNotPresent + env: + - name: PORT + value: '4003' + ports: + - containerPort: 4003 + name: http + protocol: TCP + resources: + limits: + cpu: '1' + memory: 1Gi + requests: + cpu: 10m + memory: 90Mi diff --git a/bundestag.io/admin/manifests/ingress.yaml b/bundestag.io/admin/manifests/ingress.yaml new file mode 100644 index 000000000..aefe7d9ca --- /dev/null +++ b/bundestag.io/admin/manifests/ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: admin + labels: + app: admin +spec: + ingressClassName: nginx + rules: + - host: 'admin.${var.hostname}' + http: + paths: + - backend: + service: + name: admin + port: + number: 80 + path: / + pathType: Prefix diff --git a/bundestag.io/admin/manifests/service.yaml b/bundestag.io/admin/manifests/service.yaml new file mode 100644 index 000000000..165e5fe10 --- /dev/null +++ b/bundestag.io/admin/manifests/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: admin + name: admin +spec: + type: ClusterIP + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 4003 + selector: + app: admin diff --git a/bundestag.io/api/.dockerignore b/bundestag.io/api/.dockerignore index ac165c739..02981a7dd 100644 --- a/bundestag.io/api/.dockerignore +++ b/bundestag.io/api/.dockerignore @@ -5,3 +5,6 @@ logs build *.log docker-compose* +manifests +.garden +.turbo \ No newline at end of file diff --git a/bundestag.io/api/garden.yml b/bundestag.io/api/garden.yml new file mode 100644 index 000000000..7b0ca41c6 --- /dev/null +++ b/bundestag.io/api/garden.yml @@ -0,0 +1,51 @@ +kind: Build +name: bundestag-io-api +description: Bundestag.io API +type: container +source: + path: ../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + targetStage: + $if: ${this.mode == "sync"} + $then: dev + $else: runner + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: bundestag.io + SERVICE_PATH: bundestag.io/api + +--- +kind: Deploy +name: bundestag-io-api +type: kubernetes +description: Deploy the bundestag.io API +dependencies: [build.bundestag-io-api, deploy.mongo] + +spec: + files: [./manifests/*] # <--- Tell Garden what manifests to use + + defaultTarget: # <--- This tells Garden what "target" to use for logs, code syncing and more + kind: Deployment + name: bundestag-io-api + + sync: + paths: + - containerPath: /app/bundestag.io/api/src + sourcePath: src + mode: one-way + + # Patch the K8s manifests for the api service so that we can set the correct image + patchResources: + - name: bundestag-io-api + kind: Deployment + patch: + spec: + template: + spec: + containers: + - name: bundestag-io-api + image: ${actions.build.bundestag-io-api.outputs.deploymentImageId} # <--- Reference the output from the Build action diff --git a/bundestag.io/api/manifests/deployment.yaml b/bundestag.io/api/manifests/deployment.yaml new file mode 100644 index 000000000..bb1034d51 --- /dev/null +++ b/bundestag.io/api/manifests/deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bundestag-io-api + labels: + app: bundestag-io-api +spec: + replicas: 1 + selector: + matchLabels: + app: bundestag-io-api + template: + metadata: + labels: + app: bundestag-io-api + spec: + containers: + - name: bundestag-io-api + image: democracy/bundestag.io:0.1.58 + imagePullPolicy: IfNotPresent + env: + - name: PORT + value: '4000' + - name: DB_URL + value: mongodb://democracy-mongo:27017/bundestagio + ports: + - containerPort: 4000 + name: http + protocol: TCP + resources: + limits: + cpu: '1' + memory: 1Gi + requests: + cpu: 10m + memory: 90Mi diff --git a/bundestag.io/api/manifests/ingress.yaml b/bundestag.io/api/manifests/ingress.yaml new file mode 100644 index 000000000..0178265ee --- /dev/null +++ b/bundestag.io/api/manifests/ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: bundestag-io-api + labels: + app: bundestag-io-api +spec: + ingressClassName: nginx + rules: + - host: 'bundestag-io-api.${var.hostname}' + http: + paths: + - backend: + service: + name: bundestag-io-api + port: + number: 80 + path: / + pathType: Prefix diff --git a/bundestag.io/api/manifests/service.yaml b/bundestag.io/api/manifests/service.yaml new file mode 100644 index 000000000..60606eed6 --- /dev/null +++ b/bundestag.io/api/manifests/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: bundestag-io-api + name: bundestag-io-api +spec: + type: ClusterIP + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 4000 + selector: + app: bundestag-io-api diff --git a/bundestag.io/api/package.json b/bundestag.io/api/package.json index 0c6b177cc..b54e51490 100644 --- a/bundestag.io/api/package.json +++ b/bundestag.io/api/package.json @@ -12,7 +12,8 @@ "build": "tsc", "prebuild": "pnpm run generate", "start": "node build/index.js", - "dev": "ts-node-dev --watch=resolvers,schemas --respawn src/index.ts", + "dev": "tsx watch src/index.ts", + "predev": "pnpm run generate", "lint": "pnpm lint:ts && pnpm lint:exports", "lint:es": "eslint src --ext .js,.jsx,.ts,.tsx", "lint:ts": "tsc --noEmit", @@ -50,8 +51,8 @@ "eslint-plugin-prettier": "5.0.1", "jest": "29.7.0", "prettier": "3.0.3", - "ts-node-dev": "2.0.0", "ts-unused-exports": "10.0.1", + "tsx": "^4.8.2", "typescript": "5.2.2" } } diff --git a/infra/Dockerfile.service b/infra/Dockerfile.service index 6d93470b4..b8c87c1f5 100644 --- a/infra/Dockerfile.service +++ b/infra/Dockerfile.service @@ -1,14 +1,11 @@ ARG NODE_VERSION FROM node:${NODE_VERSION}-alpine AS base -ENV PNPM_HOME="/pnpm" -ENV PATH="$PNPM_HOME:$PATH" ARG SERVICE ENV SERVICE ${SERVICE} ARG SERVICE_PATH ENV SERVICE_PATH ${SERVICE_PATH} RUN corepack enable -RUN pnpm install turbo --global FROM base AS builder RUN apk add --no-cache libc6-compat @@ -17,7 +14,7 @@ RUN apk update WORKDIR /app COPY . . -RUN turbo prune --scope=$SERVICE --docker +RUN pnpm dlx turbo prune --scope=$SERVICE --docker # FIX ERROR: failed to solve: cannot copy to non-directory: …/app/services/qr-code-handler/node_modules/tsup-config RUN find . -name "node_modules" -type d -exec rm -rf {} + @@ -51,6 +48,7 @@ WORKDIR /app # Don't run production as root RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 runuser +RUN corepack enable COPY --from=installer_prod /app . @@ -60,4 +58,4 @@ COPY --from=installer_prod /app . # USER runuser -CMD cd $SERVICE_PATH && pnpm start +CMD cd $SERVICE_PATH && npm start diff --git a/infra/democracy-api/garden.yml b/infra/democracy-api/garden.yml new file mode 100644 index 000000000..234f77ad5 --- /dev/null +++ b/infra/democracy-api/garden.yml @@ -0,0 +1,6 @@ +kind: Deploy +name: democracy-api +type: kubernetes +spec: + files: + - ./manifests/* \ No newline at end of file diff --git a/infra/democracy-api/manifests/ConfigMap.yaml b/infra/democracy-api/manifests/ConfigMap.yaml new file mode 100644 index 000000000..71446d245 --- /dev/null +++ b/infra/democracy-api/manifests/ConfigMap.yaml @@ -0,0 +1,14 @@ + +apiVersion: v1 +kind: ConfigMap +metadata: + name: democracy-api-config +data: + BUNDESTAGIO_SERVER_URL: http://bundestagio-srv:3100/ + DB_URL: mongodb://democracy-mongo:27017/democracy + DEBUG: "true" + GRAPHIQL: "true" + NODE_ENV: development + SMS_SIMULATE: "true" + SMS_VERIFICATION: "false" + STAGE: internal \ No newline at end of file diff --git a/infra/democracy-api/manifests/Deployment.yaml b/infra/democracy-api/manifests/Deployment.yaml new file mode 100644 index 000000000..4b4a39db2 --- /dev/null +++ b/infra/democracy-api/manifests/Deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: democracy-api + labels: + app: democracy-api +spec: + selector: + matchLabels: + app: democracy-api + template: + metadata: + labels: + app: democracy-api + spec: + containers: + - name: democracy-api + image: democracy/democracy-server:0.2.64 + ports: + - containerPort: 3000 + envFrom: + - configMapRef: + name: democracy-api-config + optional: false + - secretRef: + name: democracy-api-secrets + optional: false + resources: + limits: + cpu: '0.5' + memory: '512Mi' diff --git a/infra/democracy-api/manifests/Ingress.yaml b/infra/democracy-api/manifests/Ingress.yaml new file mode 100644 index 000000000..662c208e9 --- /dev/null +++ b/infra/democracy-api/manifests/Ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: democracy-api + labels: + app: democracy-api +spec: + ingressClassName: nginx + rules: + - host: 'democracy-api.${var.hostname}' + http: + paths: + - backend: + service: + name: democracy-api + port: + number: 80 + path: / + pathType: Prefix diff --git a/infra/democracy-api/manifests/Secret.yaml b/infra/democracy-api/manifests/Secret.yaml new file mode 100644 index 000000000..f9361fc04 --- /dev/null +++ b/infra/democracy-api/manifests/Secret.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: democracy-api-secrets +type: Opaque diff --git a/infra/democracy-api/manifests/Service.yaml b/infra/democracy-api/manifests/Service.yaml new file mode 100644 index 000000000..e87a8d4b9 --- /dev/null +++ b/infra/democracy-api/manifests/Service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: democracy-api + name: democracy-api +spec: + type: ClusterIP + selector: + app: democracy-api + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 3000 diff --git a/infra/mongo/garden.yml b/infra/mongo/garden.yml new file mode 100644 index 000000000..5845d8b68 --- /dev/null +++ b/infra/mongo/garden.yml @@ -0,0 +1,6 @@ +kind: Deploy +name: mongo +type: kubernetes +spec: + files: + - ./manifests/* diff --git a/infra/mongo/manifests/Deployment.yaml b/infra/mongo/manifests/Deployment.yaml new file mode 100644 index 000000000..9ba6dc48f --- /dev/null +++ b/infra/mongo/manifests/Deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: democracy-mongo + labels: + app: democracy-mongo +spec: + selector: + matchLabels: + app: democracy-mongo + template: + metadata: + labels: + app: democracy-mongo + spec: + containers: + - name: democracy-mongo + image: mongo:4 + ports: + - containerPort: 27017 + resources: + limits: + cpu: '0.5' + memory: '512Mi' diff --git a/infra/mongo/manifests/Service.yaml b/infra/mongo/manifests/Service.yaml new file mode 100644 index 000000000..156f6ed69 --- /dev/null +++ b/infra/mongo/manifests/Service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: democracy-mongo + name: democracy-mongo +spec: + type: ClusterIP + selector: + app: democracy-mongo + ports: + - name: db + protocol: TCP + port: 27017 + targetPort: 27017 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9fb35f268..d44dae36a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -211,12 +211,12 @@ importers: prettier: specifier: 3.0.3 version: 3.0.3 - ts-node-dev: - specifier: 2.0.0 - version: 2.0.0(@swc/core@1.3.93(@swc/helpers@0.5.5))(@types/node@20.12.6)(typescript@5.2.2) ts-unused-exports: specifier: 10.0.1 version: 10.0.1(typescript@5.2.2) + tsx: + specifier: ^4.8.2 + version: 4.8.2 typescript: specifier: 5.2.2 version: 5.2.2 @@ -670,8 +670,8 @@ importers: services/cron-jobs/queue-pushs-conference-week: dependencies: '@democracy-deutschland/democracy-common': - specifier: ^0.2.12 - version: 0.2.12 + specifier: workspace:* + version: link:../../../common/democracy moment: specifier: ^2.27.0 version: 2.29.4 @@ -745,8 +745,8 @@ importers: services/cron-jobs/shedule-bio-resync: dependencies: '@democracy-deutschland/democracy-common': - specifier: ^0.2.7 - version: 0.2.12 + specifier: workspace:* + version: link:../../../common/democracy devDependencies: dotenv: specifier: ^10.0.0 @@ -1811,9 +1811,6 @@ packages: '@democracy-deutschland/bundestag.io-definitions@1.0.2': resolution: {integrity: sha512-wdUnAuvN39xv+RvZWu/uWJWHOt1iDjFqFjE+5EaOBeEex6bmXpNiRCFVK25V9FfSXptjusj3FXxj2N4s5sqVKw==} - '@democracy-deutschland/democracy-common@0.2.12': - resolution: {integrity: sha512-oeNiDFCRaaEnRk92R/COEz4kr0ShgPixOK6P496orJFOHVgNWKPkHwS6f4yPZBytGz1kj/UiQbf0qrgle1RxVA==} - '@emotion/hash@0.8.0': resolution: {integrity: sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==} @@ -3321,9 +3318,6 @@ packages: '@types/body-parser@1.19.4': resolution: {integrity: sha512-N7UDG0/xiPQa2D/XrVJXjkWbpqHCd2sBaB32ggRF2l83RhPfamgKGF8gwwqyksS95qUS5ZYF9aF+lLPRlwI2UA==} - '@types/bson@4.0.5': - resolution: {integrity: sha512-vVLwMUqhYJSQ/WKcE60eFqcyuWse5fGH+NMAXHuKrUAPoryq3ATxk5o4bgYNtg5aOM4APVg7Hnb3ASqUYG0PKg==} - '@types/caseless@0.12.4': resolution: {integrity: sha512-2in/lrHRNmDvHPgyormtEralhPcN3An1gLjJzj2Bw145VBxkQ75JEXW6CTdMAwShiHQcYsl2d10IjQSdJSJz4g==} @@ -3342,9 +3336,6 @@ packages: '@types/content-type@1.1.8': resolution: {integrity: sha512-1tBhmVUeso3+ahfyaKluXe38p+94lovUZdoVfQ3OnJo9uJC42JT7CBoN3k9HYhAae+GwiBYmHu+N9FZhOG+2Pg==} - '@types/cron@1.7.3': - resolution: {integrity: sha512-iPmUXyIJG1Js+ldPYhOQcYU3kCAQ2FWrSkm1FJPoii2eYSn6wEW6onPukNTT0bfiflexNSRPl6KWmAIqS+36YA==} - '@types/cron@2.4.0': resolution: {integrity: sha512-5bBaAkqvSFBX8JMi/xCofNzG5E594TNsApMz68dLd/sQYz/HGQqgcxGHTRjOvD4G3Y+YF1Oo3S7QdCvKt1KAJQ==} deprecated: This is a stub types definition. cron provides its own type definitions, so you do not need this installed. @@ -3464,9 +3455,6 @@ packages: '@types/mocha@10.0.3': resolution: {integrity: sha512-RsOPImTriV/OE4A9qKjMtk2MnXiuLLbcO3nCXK+kvq4nr0iMfFgpjaX3MPLb6f7+EL1FGSelYvuJMV6REH+ZPQ==} - '@types/mongodb@3.6.20': - resolution: {integrity: sha512-WcdpPJCakFzcWWD9juKoZbRtQxKIMYF/JIAM4JrNHrMcnJL6/a2NWjXxW7fo9hxboxxkg+icff8d7+WIEvKgYQ==} - '@types/mongoose@5.11.97': resolution: {integrity: sha512-cqwOVYT3qXyLiGw7ueU2kX9noE8DPGRY6z8eUxudhXY8NZ7DMKYAxyZkLSevGfhCX3dO/AoX5/SO9lAzfjon0Q==} deprecated: Mongoose publishes its own types, so you do not need to install this package. @@ -4401,9 +4389,6 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} - bluebird@3.5.1: - resolution: {integrity: sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==} - bluebird@3.7.2: resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} @@ -4843,9 +4828,6 @@ packages: create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - cron@1.8.2: - resolution: {integrity: sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg==} - cron@3.1.3: resolution: {integrity: sha512-KVxeKTKYj2eNzN4ElnT6nRSbjbfhyxR92O/Jdp6SH3pc05CDJws59jBrZWEMQlxevCiE6QUTrXy+Im3vC3oD3A==} @@ -4950,14 +4932,6 @@ packages: supports-color: optional: true - debug@3.1.0: - resolution: {integrity: sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - debug@3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -7433,9 +7407,6 @@ packages: modern-ahocorasick@1.0.1: resolution: {integrity: sha512-yoe+JbhTClckZ67b2itRtistFKf8yPYelHLc7e5xAwtNAXxM6wJTUx2C7QeVSJFDzKT7bCIFyBVybPMKvmB9AA==} - moment-timezone@0.5.45: - resolution: {integrity: sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==} - moment@2.29.3: resolution: {integrity: sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==} @@ -7484,15 +7455,6 @@ packages: resolution: {tarball: https://codeload.github.com/mimani/mongoose-diff-history/tar.gz/1fb081a4308d3745ebb2646f2faeaa7ce867ca86} version: 2.1.0 - mongoose-legacy-pluralize@1.0.2: - resolution: {integrity: sha512-Yo/7qQU4/EyIS8YDFSeenIvXxZN+ld7YdV9LqFVQJzTLye8unujAWPZ4NWKfFA+RNjh+wvTWKY9Z3E5XM6ZZiQ==} - peerDependencies: - mongoose: '*' - - mongoose@5.13.22: - resolution: {integrity: sha512-p51k/c4X/MfqeQ3I1ranlDiggLzNumZrTDD9CeezHwZxt2/btf+YZD7MCe07RAY2NgFYVMayq6jMamw02Jmf9w==} - engines: {node: '>=4.0.0'} - mongoose@6.0.12: resolution: {integrity: sha512-BvsZk7zEEhb1AgQFLtxN9C+7qgy5edRuA3ZDDwHU+kHG/HM44vI6FdKV5m6HVdAUeCHHQTiVv+YQh8BRsToSHw==} engines: {node: '>=12.0.0'} @@ -7501,10 +7463,6 @@ packages: resolution: {integrity: sha512-DTxNZomBcTWlrMW76jy1wvV37X/cNNxPW1y2Jzd4DZkAaC5ZGsm8bfGfNOthcDuRJujXLqiuS6o3Tpy0JEoh7g==} engines: {node: '>=4.0.0'} - mquery@3.2.5: - resolution: {integrity: sha512-VjOKHHgU84wij7IUoZzFRU07IAxd5kWJaDmyUzQlbjHjyoeK5TNeeo8ZsFDtTYnSgpW6n/nMNIHvE3u8Lbrf4A==} - engines: {node: '>=4.0.0'} - mquery@4.0.0: resolution: {integrity: sha512-nGjm89lHja+T/b8cybAby6H0YgA4qYC/lx6UlwvHGqvTq8bDaNeCwl1sY8uRELrNbVWJzIihxVd+vphGGn1vBw==} engines: {node: '>=12.0.0'} @@ -7786,10 +7744,6 @@ packages: optimism@0.18.0: resolution: {integrity: sha512-tGn8+REwLRNFnb9WmcY5IfpOqeX2kpaYJ1s6Ae3mn12AeydLkR3j+jSCmVQFoXqU8D41PAJ1RG1rCRNWmNZVmQ==} - optional-require@1.0.3: - resolution: {integrity: sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA==} - engines: {node: '>=4'} - optional-require@1.1.8: resolution: {integrity: sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA==} engines: {node: '>=4'} @@ -9354,13 +9308,6 @@ packages: ts-log@2.2.5: resolution: {integrity: sha512-PGcnJoTBnVGy6yYNFxWVNkdcAuAMstvutN9MgDJIV6L0oG8fB+ZNNy1T+wJzah8RPGor1mZuPQkVfXNDpy9eHA==} - ts-mongoose@0.0.21: - resolution: {integrity: sha512-pbRN2JJY9N7Z5ehyMo6PJ7hlftRdELzO5nh/XvS1y31TDbeefc/GjM7C7jWb6ESZz+KlOmYDUpAT/L8i4wFbmQ==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. - peerDependencies: - '@types/mongoose': ^5.5.21 - mongoose: ^5.7.5 - ts-mongoose@0.0.24: resolution: {integrity: sha512-BaiyNFqieRqEBnvj2pgWB6/eQJRm7U18kAzDJ92/sfcFx1aQBZeZztzutxr49VVlUvPHxukleIoU/t3VMCBsSA==} peerDependencies: @@ -9527,6 +9474,11 @@ packages: peerDependencies: typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta' + tsx@4.8.2: + resolution: {integrity: sha512-hmmzS4U4mdy1Cnzpl/NQiPUC2k34EcNSTZYVJThYKhdqTwuBeF+4cG9KUK/PFQ7KHaAaYwqlb7QfmsE2nuj+WA==} + engines: {node: '>=18.0.0'} + hasBin: true + tty@1.0.1: resolution: {integrity: sha512-yCPqGIuidycVkRigBDshlGDLKu9+p4JKtBQijtYcI7ZnWwak56vwT7y7dGGTcxG+ecjxgWQOPWuvlePmxC5S2w==} @@ -11150,24 +11102,6 @@ snapshots: dependencies: typescript: 3.9.10 - '@democracy-deutschland/democracy-common@0.2.12': - dependencies: - '@democracy-deutschland/bundestag.io-definitions': 1.0.2 - '@types/cron': 1.7.3 - '@types/mongoose': 5.11.97 - cron: 1.8.2 - mongoose: 5.13.22 - ts-mongoose: 0.0.21(@types/mongoose@5.11.97)(mongoose@5.13.22) - ts-unused-exports: 6.3.0 - transitivePeerDependencies: - - aws4 - - bson-ext - - kerberos - - mongodb-client-encryption - - mongodb-extjson - - snappy - - supports-color - '@emotion/hash@0.8.0': {} '@emotion/hash@0.9.1': {} @@ -13052,10 +12986,6 @@ snapshots: '@types/connect': 3.4.37 '@types/node': 20.12.6 - '@types/bson@4.0.5': - dependencies: - '@types/node': 20.12.6 - '@types/caseless@0.12.4': {} '@types/chai@4.3.9': {} @@ -13074,11 +13004,6 @@ snapshots: '@types/content-type@1.1.8': {} - '@types/cron@1.7.3': - dependencies: - '@types/node': 20.12.6 - moment: 2.29.4 - '@types/cron@2.4.0': dependencies: cron: 3.1.3 @@ -13220,11 +13145,6 @@ snapshots: '@types/mocha@10.0.3': {} - '@types/mongodb@3.6.20': - dependencies: - '@types/bson': 4.0.5 - '@types/node': 20.12.6 - '@types/mongoose@5.11.97': dependencies: mongoose: 6.0.12 @@ -14664,8 +14584,6 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 - bluebird@3.5.1: {} - bluebird@3.7.2: optional: true @@ -15280,10 +15198,6 @@ snapshots: create-require@1.1.1: {} - cron@1.8.2: - dependencies: - moment-timezone: 0.5.45 - cron@3.1.3: dependencies: '@types/luxon': 3.3.3 @@ -15397,10 +15311,6 @@ snapshots: dependencies: ms: 2.0.0 - debug@3.1.0: - dependencies: - ms: 2.0.0 - debug@3.2.7(supports-color@5.5.0): dependencies: ms: 2.1.3 @@ -18848,10 +18758,6 @@ snapshots: modern-ahocorasick@1.0.1: {} - moment-timezone@0.5.45: - dependencies: - moment: 2.29.4 - moment@2.29.3: {} moment@2.29.4: {} @@ -18894,35 +18800,6 @@ snapshots: omit-deep: https://codeload.github.com/izigibran/omit-deep/tar.gz/18cd03c66cd267735f09741254b638efba56dce6 power-assign: 0.2.10 - mongoose-legacy-pluralize@1.0.2(mongoose@5.13.22): - dependencies: - mongoose: 5.13.22 - - mongoose@5.13.22: - dependencies: - '@types/bson': 4.0.5 - '@types/mongodb': 3.6.20 - bson: 1.1.6 - kareem: 2.3.2 - mongodb: 3.7.4 - mongoose-legacy-pluralize: 1.0.2(mongoose@5.13.22) - mpath: 0.8.4 - mquery: 3.2.5 - ms: 2.1.2 - optional-require: 1.0.3 - regexp-clone: 1.0.0 - safe-buffer: 5.2.1 - sift: 13.5.2 - sliced: 1.0.1 - transitivePeerDependencies: - - aws4 - - bson-ext - - kerberos - - mongodb-client-encryption - - mongodb-extjson - - snappy - - supports-color - mongoose@6.0.12: dependencies: bson: 4.7.2 @@ -18939,16 +18816,6 @@ snapshots: mpath@0.8.4: {} - mquery@3.2.5: - dependencies: - bluebird: 3.5.1 - debug: 3.1.0 - regexp-clone: 1.0.0 - safe-buffer: 5.1.2 - sliced: 1.0.1 - transitivePeerDependencies: - - supports-color - mquery@4.0.0: dependencies: debug: 4.3.4(supports-color@8.1.1) @@ -19250,8 +19117,6 @@ snapshots: '@wry/trie': 0.4.3 tslib: 2.6.2 - optional-require@1.0.3: {} - optional-require@1.1.8: dependencies: require-at: 1.0.6 @@ -21088,11 +20953,6 @@ snapshots: ts-log@2.2.5: {} - ts-mongoose@0.0.21(@types/mongoose@5.11.97)(mongoose@5.13.22): - dependencies: - '@types/mongoose': 5.11.97 - mongoose: 5.13.22 - ts-mongoose@0.0.24(@types/mongoose@5.11.97)(mongoose@6.0.12): dependencies: '@types/mongoose': 5.11.97 @@ -21598,6 +21458,13 @@ snapshots: tslib: 1.14.1 typescript: 5.4.4 + tsx@4.8.2: + dependencies: + esbuild: 0.20.2 + get-tsconfig: 4.7.3 + optionalDependencies: + fsevents: 2.3.3 + tty@1.0.1: {} tunnel-agent@0.6.0: diff --git a/project.garden.yml b/project.garden.yml new file mode 100644 index 000000000..aeb676bfb --- /dev/null +++ b/project.garden.yml @@ -0,0 +1,24 @@ +apiVersion: garden.io/v1 +kind: Project +name: democracy-development + +defaultEnvironment: local + +variables: + userNamespace: democracy-local + +environments: + - name: local + defaultNamespace: ${var.userNamespace} + variables: + # hostname: local.democracy-app.de + hostname: local.app.garden + - name: local-prod + defaultNamespace: ${var.userNamespace} + variables: + # hostname: local.democracy-app.de + hostname: local.app.garden + +providers: + - name: local-kubernetes + environments: [local, local-prod] diff --git a/services/cron-jobs/cleanup-push-queue/garden.yml b/services/cron-jobs/cleanup-push-queue/garden.yml new file mode 100644 index 000000000..d03a7b952 --- /dev/null +++ b/services/cron-jobs/cleanup-push-queue/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: cleanup-push-queue +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: cleanup-push-queue + SERVICE_PATH: services/cron-jobs/cleanup-push-queue + +--- +kind: Deploy +name: cleanup-push-queue +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.cleanup-push-queue, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + +spec: + files: [./manifests/*] + + patchResources: + - name: cleanup-push-queue + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: cleanup-push-queue + image: ${actions.build.cleanup-push-queue.outputs.deploymentImageId} diff --git a/services/cron-jobs/cleanup-push-queue/manifests/CronJob.yaml b/services/cron-jobs/cleanup-push-queue/manifests/CronJob.yaml new file mode 100644 index 000000000..b895103e8 --- /dev/null +++ b/services/cron-jobs/cleanup-push-queue/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cleanup-push-queue + labels: + app: cleanup-push-queue +spec: + schedule: '0 12 * * SUN' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: cleanup-push-queue + image: democracy/cleanup-push-queue:1.0.4 + imagePullPolicy: IfNotPresent + env: + - name: DB_URL + value: mongodb://democracy-mongo:27017/democracy diff --git a/services/cron-jobs/crawler/.env b/services/cron-jobs/crawler/.env index 19bd35121..aaef55d64 100644 --- a/services/cron-jobs/crawler/.env +++ b/services/cron-jobs/crawler/.env @@ -1,7 +1,7 @@ IMPORT_PROCEDURES_START_CURSOR=* IMPORT_PROCEDURES_FILTER_TYPES=Antrag,Gesetzgebung IMPORT_PROCEDURES_START_CURSOR=* -DEBUG=* +DEBUG=bundestag-io:import-procedures:log IMPORT_PROCEDURES_CHUNK_SIZE=50 IMPORT_PROCEDURES_CHUNK_ROUNDS=100000 IMPORT_PROCEDURES_FILTER_AFTER=2024-01-01 \ No newline at end of file diff --git a/services/cron-jobs/crawler/garden.yml b/services/cron-jobs/crawler/garden.yml new file mode 100644 index 000000000..0981c9912 --- /dev/null +++ b/services/cron-jobs/crawler/garden.yml @@ -0,0 +1,54 @@ +kind: Build +name: import-procedures +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: crawler + SERVICE_PATH: services/cron-jobs/crawler + +--- +kind: Deploy +name: import-procedures +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.import-procedures, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +varfiles: + - .env + - .env.local + +variables: + DB_URL: mongodb://democracy-mongo:27017/bundestagio + +spec: + files: + - ./manifests/* + + patchResources: + - name: import-procedures + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: import-procedures + image: ${actions.build.import-procedures.outputs.deploymentImageId} # <--- Reference the output from the Build action + + - name: dip-api-token + kind: Secret + patch: + data: + DIP_API_KEY: ${base64Encode(var.DIP_API_KEY)} diff --git a/services/cron-jobs/crawler/manifests/ConfigMap.yaml b/services/cron-jobs/crawler/manifests/ConfigMap.yaml new file mode 100644 index 000000000..0665aac77 --- /dev/null +++ b/services/cron-jobs/crawler/manifests/ConfigMap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: crawler-config +data: + DB_URL: ${var.DB_URL} + DEBUG: '${var.DEBUG}' + IMPORT_PROCEDURES_CHUNK_ROUNDS: '${var.IMPORT_PROCEDURES_CHUNK_ROUNDS}' + IMPORT_PROCEDURES_CHUNK_SIZE: '${var.IMPORT_PROCEDURES_CHUNK_SIZE}' + IMPORT_PROCEDURES_FILTER_AFTER: '${var.IMPORT_PROCEDURES_FILTER_AFTER}' + IMPORT_PROCEDURES_FILTER_TYPES: '${var.IMPORT_PROCEDURES_FILTER_TYPES}' + IMPORT_PROCEDURES_START_CURSOR: '${var.IMPORT_PROCEDURES_START_CURSOR}' diff --git a/services/cron-jobs/crawler/manifests/CronJob.yaml b/services/cron-jobs/crawler/manifests/CronJob.yaml new file mode 100644 index 000000000..0d0186cef --- /dev/null +++ b/services/cron-jobs/crawler/manifests/CronJob.yaml @@ -0,0 +1,27 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: import-procedures + labels: + app: import-procedures +spec: + schedule: '0 1 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: import-procedures + image: democracy/crawler:1.0.17 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: crawler-config + - secretRef: + name: dip-api-token + optional: false diff --git a/services/cron-jobs/crawler/manifests/Secret.yaml b/services/cron-jobs/crawler/manifests/Secret.yaml new file mode 100644 index 000000000..52c753cb0 --- /dev/null +++ b/services/cron-jobs/crawler/manifests/Secret.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: dip-api-token +type: Opaque diff --git a/services/cron-jobs/import-conference-week-details/garden.yml b/services/cron-jobs/import-conference-week-details/garden.yml new file mode 100644 index 000000000..556cbf634 --- /dev/null +++ b/services/cron-jobs/import-conference-week-details/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: import-conference-week-details +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: import-conference-week-details + SERVICE_PATH: services/cron-jobs/import-conference-week-details + +--- +kind: Deploy +name: import-conference-week-details +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.import-conference-week-details, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/bundestagio + +spec: + files: [./manifests/*] + + patchResources: + - name: import-conference-week-details + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: import-conference-week-details + image: ${actions.build.import-conference-week-details.outputs.deploymentImageId} diff --git a/services/cron-jobs/import-conference-week-details/manifests/ConfigMap.yaml b/services/cron-jobs/import-conference-week-details/manifests/ConfigMap.yaml new file mode 100644 index 000000000..3fa1968a4 --- /dev/null +++ b/services/cron-jobs/import-conference-week-details/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: import-conference-week-details +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/import-conference-week-details/manifests/CronJob.yaml b/services/cron-jobs/import-conference-week-details/manifests/CronJob.yaml new file mode 100644 index 000000000..7f1a0da66 --- /dev/null +++ b/services/cron-jobs/import-conference-week-details/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: import-conference-week-details + labels: + app: import-conference-week-details +spec: + schedule: '*/30 * * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: import-conference-week-details + image: democracy/import-conference-week-details:0.1.26 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: import-conference-week-details diff --git a/services/cron-jobs/import-deputy-profiles/garden.yml b/services/cron-jobs/import-deputy-profiles/garden.yml new file mode 100644 index 000000000..2f46aff39 --- /dev/null +++ b/services/cron-jobs/import-deputy-profiles/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: import-deputy-profiles +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: import-deputy-profiles + SERVICE_PATH: services/cron-jobs/import-deputy-profiles + +--- +kind: Deploy +name: import-deputy-profiles +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.import-deputy-profiles, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/bundestagio + +spec: + files: [./manifests/*] + + patchResources: + - name: import-deputy-profiles + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: import-deputy-profiles + image: ${actions.build.import-deputy-profiles.outputs.deploymentImageId} diff --git a/services/cron-jobs/import-deputy-profiles/manifests/ConfigMap.yaml b/services/cron-jobs/import-deputy-profiles/manifests/ConfigMap.yaml new file mode 100644 index 000000000..c33d55688 --- /dev/null +++ b/services/cron-jobs/import-deputy-profiles/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: import-deputy-profiles +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/import-deputy-profiles/manifests/CronJob.yaml b/services/cron-jobs/import-deputy-profiles/manifests/CronJob.yaml new file mode 100644 index 000000000..859acbd5c --- /dev/null +++ b/services/cron-jobs/import-deputy-profiles/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: import-deputy-profiles + labels: + app: import-deputy-profiles +spec: + schedule: '0 5 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: import-deputy-profiles + image: democracy/import-deputy-profiles:0.1.12 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: import-deputy-profiles diff --git a/services/cron-jobs/import-named-poll-deputies/garden.yml b/services/cron-jobs/import-named-poll-deputies/garden.yml new file mode 100644 index 000000000..b2d4b148e --- /dev/null +++ b/services/cron-jobs/import-named-poll-deputies/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: import-named-poll-deputies +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: import-named-poll-deputies + SERVICE_PATH: services/cron-jobs/import-named-poll-deputies + +--- +kind: Deploy +name: import-named-poll-deputies +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.import-named-poll-deputies, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/bundestagio + +spec: + files: [./manifests/*] + + patchResources: + - name: import-named-poll-deputies + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: import-named-poll-deputies + image: ${actions.build.import-named-poll-deputies.outputs.deploymentImageId} diff --git a/services/cron-jobs/import-named-poll-deputies/manifests/ConfigMap.yaml b/services/cron-jobs/import-named-poll-deputies/manifests/ConfigMap.yaml new file mode 100644 index 000000000..c950c1e6d --- /dev/null +++ b/services/cron-jobs/import-named-poll-deputies/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: import-named-poll-deputies +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/import-named-poll-deputies/manifests/CronJob.yaml b/services/cron-jobs/import-named-poll-deputies/manifests/CronJob.yaml new file mode 100644 index 000000000..2630164bd --- /dev/null +++ b/services/cron-jobs/import-named-poll-deputies/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: import-named-poll-deputies + labels: + app: import-named-poll-deputies +spec: + schedule: '45 1 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: import-named-poll-deputies + image: democracy/import-named-poll-deputies:0.1.8 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: import-named-poll-deputies diff --git a/services/cron-jobs/import-named-polls/garden.yml b/services/cron-jobs/import-named-polls/garden.yml new file mode 100644 index 000000000..af862b576 --- /dev/null +++ b/services/cron-jobs/import-named-polls/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: import-named-polls +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: import-named-polls + SERVICE_PATH: services/cron-jobs/import-named-polls + +--- +kind: Deploy +name: import-named-polls +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.import-named-polls, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/bundestagio + +spec: + files: [./manifests/*] + + patchResources: + - name: import-named-polls + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: import-named-polls + image: ${actions.build.import-named-polls.outputs.deploymentImageId} diff --git a/services/cron-jobs/import-named-polls/manifests/ConfigMap.yaml b/services/cron-jobs/import-named-polls/manifests/ConfigMap.yaml new file mode 100644 index 000000000..0d1b5c87b --- /dev/null +++ b/services/cron-jobs/import-named-polls/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: import-named-polls +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/import-named-polls/manifests/CronJob.yaml b/services/cron-jobs/import-named-polls/manifests/CronJob.yaml new file mode 100644 index 000000000..5900dea62 --- /dev/null +++ b/services/cron-jobs/import-named-polls/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: import-named-polls + labels: + app: import-named-polls +spec: + schedule: '15 1 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: import-named-polls + image: democracy/import-named-polls:0.1.17 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: import-named-polls diff --git a/services/cron-jobs/import-plenary-minutes/garden.yml b/services/cron-jobs/import-plenary-minutes/garden.yml new file mode 100644 index 000000000..4684a6dd9 --- /dev/null +++ b/services/cron-jobs/import-plenary-minutes/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: import-plenary-minutes +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: import-plenary-minutes + SERVICE_PATH: services/cron-jobs/import-plenary-minutes + +--- +kind: Deploy +name: import-plenary-minutes +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.import-plenary-minutes, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/bundestagio + +spec: + files: [./manifests/*] + + patchResources: + - name: import-plenary-minutes + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: import-plenary-minutes + image: ${actions.build.import-plenary-minutes.outputs.deploymentImageId} diff --git a/services/cron-jobs/import-plenary-minutes/manifests/ConfigMap.yaml b/services/cron-jobs/import-plenary-minutes/manifests/ConfigMap.yaml new file mode 100644 index 000000000..1360fa247 --- /dev/null +++ b/services/cron-jobs/import-plenary-minutes/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: import-plenary-minutes +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/import-plenary-minutes/manifests/CronJob.yaml b/services/cron-jobs/import-plenary-minutes/manifests/CronJob.yaml new file mode 100644 index 000000000..dec761c1f --- /dev/null +++ b/services/cron-jobs/import-plenary-minutes/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: import-plenary-minutes + labels: + app: import-plenary-minutes +spec: + schedule: '15 1 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: import-plenary-minutes + image: democracy/import-plenary-minutes:0.1.6 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: import-plenary-minutes diff --git a/services/cron-jobs/index-sync-bundestagio/garden.yml b/services/cron-jobs/index-sync-bundestagio/garden.yml new file mode 100644 index 000000000..dbe044b0b --- /dev/null +++ b/services/cron-jobs/index-sync-bundestagio/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: index-sync-bundestagio +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: index-sync-bundestagio + SERVICE_PATH: services/cron-jobs/index-sync-bundestagio + +--- +kind: Deploy +name: index-sync-bundestagio +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.index-sync-bundestagio, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/bundestagio + +spec: + files: [./manifests/*] + + patchResources: + - name: index-sync-bundestagio + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: index-sync-bundestagio + image: ${actions.build.index-sync-bundestagio.outputs.deploymentImageId} diff --git a/services/cron-jobs/index-sync-bundestagio/manifests/ConfigMap.yaml b/services/cron-jobs/index-sync-bundestagio/manifests/ConfigMap.yaml new file mode 100644 index 000000000..a6770aa9c --- /dev/null +++ b/services/cron-jobs/index-sync-bundestagio/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: index-sync-bundestagio +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/index-sync-bundestagio/manifests/CronJob.yaml b/services/cron-jobs/index-sync-bundestagio/manifests/CronJob.yaml new file mode 100644 index 000000000..d035ee2a7 --- /dev/null +++ b/services/cron-jobs/index-sync-bundestagio/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: index-sync-bundestagio + labels: + app: index-sync-bundestagio +spec: + schedule: '0 7 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: index-sync-bundestagio + image: democracy/import-plenary-minutes:0.1.6 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: index-sync-bundestagio diff --git a/services/cron-jobs/index-sync-democracy/garden.yml b/services/cron-jobs/index-sync-democracy/garden.yml new file mode 100644 index 000000000..fc441651c --- /dev/null +++ b/services/cron-jobs/index-sync-democracy/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: index-sync-democracy +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: index-sync-democracy + SERVICE_PATH: services/cron-jobs/index-sync-democracy + +--- +kind: Deploy +name: index-sync-democracy +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.index-sync-democracy, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + +spec: + files: [./manifests/*] + + patchResources: + - name: index-sync-democracy + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: index-sync-democracy + image: ${actions.build.index-sync-democracy.outputs.deploymentImageId} diff --git a/services/cron-jobs/index-sync-democracy/manifests/ConfigMap.yaml b/services/cron-jobs/index-sync-democracy/manifests/ConfigMap.yaml new file mode 100644 index 000000000..3c27233f6 --- /dev/null +++ b/services/cron-jobs/index-sync-democracy/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: index-sync-democracy +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/index-sync-democracy/manifests/CronJob.yaml b/services/cron-jobs/index-sync-democracy/manifests/CronJob.yaml new file mode 100644 index 000000000..171b5c292 --- /dev/null +++ b/services/cron-jobs/index-sync-democracy/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: index-sync-democracy + labels: + app: index-sync-democracy +spec: + schedule: '0 7 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: index-sync-democracy + image: democracy/index-sync-democracy:0.1.4 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: index-sync-democracy diff --git a/services/cron-jobs/push-send-queued/.env.example b/services/cron-jobs/push-send-queued/.env.example index c759ade01..aba77dd76 100644 --- a/services/cron-jobs/push-send-queued/.env.example +++ b/services/cron-jobs/push-send-queued/.env.example @@ -1,8 +1,4 @@ DB_URL=mongodb://localhost/democracy -# NODE_ENV=development -APN_TOPIC=de.democracy-deutschland.clientapp.internal -GORUSH_URL=http://localhost:8088 -APPLE_TEAMID=A4B84UJD7M SERVICE_ACCOUNT_CLIENT_EMAIL= SERVICE_ACCOUNT_PRIVATE_KEY= SERVICE_ACCOUNT_PROJECT_ID= \ No newline at end of file diff --git a/services/cron-jobs/push-send-queued/garden.yml b/services/cron-jobs/push-send-queued/garden.yml new file mode 100644 index 000000000..f59ea2768 --- /dev/null +++ b/services/cron-jobs/push-send-queued/garden.yml @@ -0,0 +1,54 @@ +kind: Build +name: push-send-queued +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: push-send-queued + SERVICE_PATH: services/cron-jobs/push-send-queued + +--- +kind: Deploy +name: push-send-queued +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.push-send-queued, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +varfiles: + - .env + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + +spec: + files: [./manifests/*] + + patchResources: + - name: push-send-queued + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: push-send-queued + image: ${actions.build.push-send-queued.outputs.deploymentImageId} + - name: push-send-queued + kind: Secret + patch: + data: + SERVICE_ACCOUNT_CLIENT_EMAIL: ${base64Encode(var.SERVICE_ACCOUNT_CLIENT_EMAIL)} + SERVICE_ACCOUNT_PRIVATE_KEY: ${base64Encode(var.SERVICE_ACCOUNT_PRIVATE_KEY)} + SERVICE_ACCOUNT_PROJECT_ID: ${base64Encode(var.SERVICE_ACCOUNT_PROJECT_ID)} + diff --git a/services/cron-jobs/push-send-queued/manifests/ConfigMap.yaml b/services/cron-jobs/push-send-queued/manifests/ConfigMap.yaml new file mode 100644 index 000000000..b0e33c0d7 --- /dev/null +++ b/services/cron-jobs/push-send-queued/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: push-send-queued +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/push-send-queued/manifests/CronJob.yaml b/services/cron-jobs/push-send-queued/manifests/CronJob.yaml new file mode 100644 index 000000000..606af0c45 --- /dev/null +++ b/services/cron-jobs/push-send-queued/manifests/CronJob.yaml @@ -0,0 +1,27 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: push-send-queued + labels: + app: push-send-queued +spec: + schedule: '*/1 8-20 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: push-send-queued + image: democracy/push-send-queued:0.1.24 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: push-send-queued + - secretRef: + name: push-send-queued + optional: false diff --git a/services/cron-jobs/push-send-queued/manifests/Secret.yaml b/services/cron-jobs/push-send-queued/manifests/Secret.yaml new file mode 100644 index 000000000..0f45d5507 --- /dev/null +++ b/services/cron-jobs/push-send-queued/manifests/Secret.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: push-send-queued +type: Opaque diff --git a/services/cron-jobs/queue-pushs-conference-week/.env.local b/services/cron-jobs/queue-pushs-conference-week/.env.local index 873d1170b..0e0182493 100644 --- a/services/cron-jobs/queue-pushs-conference-week/.env.local +++ b/services/cron-jobs/queue-pushs-conference-week/.env.local @@ -1,2 +1 @@ DB_URL=mongodb://localhost/democracy -BUNDESTAGIO_SERVER_URL=http://localhost:4000 diff --git a/services/cron-jobs/queue-pushs-conference-week/Dockerfile b/services/cron-jobs/queue-pushs-conference-week/Dockerfile deleted file mode 100644 index d74cd5f48..000000000 --- a/services/cron-jobs/queue-pushs-conference-week/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM node:12-alpine AS BUILD_IMAGE - -# install next-optimized-images requirements -RUN apk --no-cache update \ - && apk --no-cache add curl bash \ - && rm -fr /var/cache/apk/* - -# install node-prune (https://github.com/tj/node-prune) -RUN curl -sfL https://install.goreleaser.com/github.com/tj/node-prune.sh | bash -s -- -b /usr/local/bin - -WORKDIR /app -COPY package.json pnpm-lock.yaml ./ -RUN pnpm --frozen-lockfile -COPY . . - -RUN pnpm build - -RUN npm prune --production - -# run node prune -RUN /usr/local/bin/node-prune - -FROM node:12-alpine - -WORKDIR /app - -COPY . . - -# copy from build image -COPY --from=BUILD_IMAGE /app/build ./build -COPY --from=BUILD_IMAGE /app/node_modules ./node_modules - -ENV NODE_ENV=production - -ENTRYPOINT [ "pnpm", "start" ] \ No newline at end of file diff --git a/services/cron-jobs/queue-pushs-conference-week/Dockerfile.dev b/services/cron-jobs/queue-pushs-conference-week/Dockerfile.dev deleted file mode 100644 index 36cdcc544..000000000 --- a/services/cron-jobs/queue-pushs-conference-week/Dockerfile.dev +++ /dev/null @@ -1,15 +0,0 @@ -FROM node:12-alpine - -RUN apk --no-cache update \ - && apk --no-cache add git \ - && rm -fr /var/cache/apk/* - -WORKDIR /app -COPY package.json pnpm-lock.yaml ./ -RUN pnpm --frozen-lockfile - -COPY . . - -ENV NODE_ENV=development - -ENTRYPOINT [ "pnpm", "dev" ] \ No newline at end of file diff --git a/services/cron-jobs/queue-pushs-conference-week/garden.yml b/services/cron-jobs/queue-pushs-conference-week/garden.yml new file mode 100644 index 000000000..91880c1ea --- /dev/null +++ b/services/cron-jobs/queue-pushs-conference-week/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: queue-pushs-conference-week +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: queue-pushs-conference-week + SERVICE_PATH: services/cron-jobs/queue-pushs-conference-week + +--- +kind: Deploy +name: queue-pushs-conference-week +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.queue-pushs-conference-week, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + +spec: + files: [./manifests/*] + + patchResources: + - name: queue-pushs-conference-week + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: queue-pushs-conference-week + image: ${actions.build.queue-pushs-conference-week.outputs.deploymentImageId} diff --git a/services/cron-jobs/queue-pushs-conference-week/manifests/ConfigMap.yaml b/services/cron-jobs/queue-pushs-conference-week/manifests/ConfigMap.yaml new file mode 100644 index 000000000..5b6cdd1fb --- /dev/null +++ b/services/cron-jobs/queue-pushs-conference-week/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: queue-pushs-conference-week +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/queue-pushs-conference-week/manifests/CronJob.yaml b/services/cron-jobs/queue-pushs-conference-week/manifests/CronJob.yaml new file mode 100644 index 000000000..b776e5257 --- /dev/null +++ b/services/cron-jobs/queue-pushs-conference-week/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: queue-pushs-conference-week + labels: + app: queue-pushs-conference-week +spec: + schedule: '0 12 * * SUN' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: queue-pushs-conference-week + image: democracy/queue-pushs-conference-week:0.1.9 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: queue-pushs-conference-week diff --git a/services/cron-jobs/queue-pushs-conference-week/package.json b/services/cron-jobs/queue-pushs-conference-week/package.json index 95029f7de..8d68f2207 100644 --- a/services/cron-jobs/queue-pushs-conference-week/package.json +++ b/services/cron-jobs/queue-pushs-conference-week/package.json @@ -12,7 +12,7 @@ "start": "node ./build/index.js" }, "dependencies": { - "@democracy-deutschland/democracy-common": "^0.2.12", + "@democracy-deutschland/democracy-common": "workspace:*", "moment": "^2.27.0" }, "devDependencies": { diff --git a/services/cron-jobs/queue-pushs-conference-week/src/index.ts b/services/cron-jobs/queue-pushs-conference-week/src/index.ts index d758e1729..ef4166f3e 100644 --- a/services/cron-jobs/queue-pushs-conference-week/src/index.ts +++ b/services/cron-jobs/queue-pushs-conference-week/src/index.ts @@ -91,9 +91,9 @@ const start = async () => { (async () => { console.info('START'); - console.info('process.env', process.env.BUNDESTAGIO_SERVER_URL, process.env.DB_URL); - if (!process.env.BUNDESTAGIO_SERVER_URL) { - throw new Error('you have to set environment variable: BUNDESTAGIO_SERVER_URL & DB_URL'); + console.info('process.env', process.env.DB_URL); + if (!process.env.DB_URL) { + throw new Error('you have to set environment variable: DB_URL'); } await mongoConnect(); console.log('procedures', await ProcedureModel.countDocuments({})); diff --git a/services/cron-jobs/queue-pushs-vote-conference-week/.env.local b/services/cron-jobs/queue-pushs-vote-conference-week/.env.local index 873d1170b..0e0182493 100644 --- a/services/cron-jobs/queue-pushs-vote-conference-week/.env.local +++ b/services/cron-jobs/queue-pushs-vote-conference-week/.env.local @@ -1,2 +1 @@ DB_URL=mongodb://localhost/democracy -BUNDESTAGIO_SERVER_URL=http://localhost:4000 diff --git a/services/cron-jobs/queue-pushs-vote-conference-week/garden.yml b/services/cron-jobs/queue-pushs-vote-conference-week/garden.yml new file mode 100644 index 000000000..ce12008df --- /dev/null +++ b/services/cron-jobs/queue-pushs-vote-conference-week/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: queue-pushs-vote-conference-week +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: queue-pushs-vote-conference-week + SERVICE_PATH: services/cron-jobs/queue-pushs-vote-conference-week + +--- +kind: Deploy +name: queue-pushs-vote-conference-week +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.queue-pushs-vote-conference-week, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + +spec: + files: [./manifests/*] + + patchResources: + - name: queue-pushs-vote-conference-week + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: queue-pushs-vote-conference-week + image: ${actions.build.queue-pushs-vote-conference-week.outputs.deploymentImageId} diff --git a/services/cron-jobs/queue-pushs-vote-conference-week/manifests/ConfigMap.yaml b/services/cron-jobs/queue-pushs-vote-conference-week/manifests/ConfigMap.yaml new file mode 100644 index 000000000..492da515f --- /dev/null +++ b/services/cron-jobs/queue-pushs-vote-conference-week/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: queue-pushs-vote-conference-week +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/queue-pushs-vote-conference-week/manifests/CronJob.yaml b/services/cron-jobs/queue-pushs-vote-conference-week/manifests/CronJob.yaml new file mode 100644 index 000000000..a951542b5 --- /dev/null +++ b/services/cron-jobs/queue-pushs-vote-conference-week/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: queue-pushs-vote-conference-week + labels: + app: queue-pushs-vote-conference-week +spec: + schedule: '0 2 * * MON-FRI' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: queue-pushs-vote-conference-week + image: democracy/queue-pushs-vote-conference-week:0.1.7 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: queue-pushs-vote-conference-week diff --git a/services/cron-jobs/queue-pushs-vote-conference-week/src/index.ts b/services/cron-jobs/queue-pushs-vote-conference-week/src/index.ts index 0a6777808..1a39e3344 100644 --- a/services/cron-jobs/queue-pushs-vote-conference-week/src/index.ts +++ b/services/cron-jobs/queue-pushs-vote-conference-week/src/index.ts @@ -144,9 +144,9 @@ const start = async () => { (async () => { console.info('START'); - console.info('process.env', process.env.BUNDESTAGIO_SERVER_URL, process.env.DB_URL); - if (!process.env.BUNDESTAGIO_SERVER_URL) { - throw new Error('you have to set environment variable: BUNDESTAGIO_SERVER_URL & DB_URL'); + console.info('process.env', process.env.DB_URL); + if (!process.env.DB_URL) { + throw new Error('you have to set environment variable: DB_URL'); } await mongoConnect(); console.log('procedures', await ProcedureModel.countDocuments({})); diff --git a/services/cron-jobs/queue-pushs-vote-top-100/garden.yml b/services/cron-jobs/queue-pushs-vote-top-100/garden.yml new file mode 100644 index 000000000..a48c35aab --- /dev/null +++ b/services/cron-jobs/queue-pushs-vote-top-100/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: queue-pushs-vote-top-100 +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: queue-pushs-vote-top-100 + SERVICE_PATH: services/cron-jobs/queue-pushs-vote-top-100 + +--- +kind: Deploy +name: queue-pushs-vote-top-100 +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.queue-pushs-vote-top-100, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + +spec: + files: [./manifests/*] + + patchResources: + - name: queue-pushs-vote-top-100 + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: queue-pushs-vote-top-100 + image: ${actions.build.queue-pushs-vote-top-100.outputs.deploymentImageId} diff --git a/services/cron-jobs/queue-pushs-vote-top-100/manifests/ConfigMap.yaml b/services/cron-jobs/queue-pushs-vote-top-100/manifests/ConfigMap.yaml new file mode 100644 index 000000000..3952b5d4b --- /dev/null +++ b/services/cron-jobs/queue-pushs-vote-top-100/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: queue-pushs-vote-top-100 +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/queue-pushs-vote-top-100/manifests/CronJob.yaml b/services/cron-jobs/queue-pushs-vote-top-100/manifests/CronJob.yaml new file mode 100644 index 000000000..f8b99860a --- /dev/null +++ b/services/cron-jobs/queue-pushs-vote-top-100/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: queue-pushs-vote-top-100 + labels: + app: queue-pushs-vote-top-100 +spec: + schedule: '0 2 * * MON-FRI' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: queue-pushs-vote-top-100 + image: democracy/queue-pushs-vote-top-100:0.1.14 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: queue-pushs-vote-top-100 diff --git a/services/cron-jobs/shedule-bio-resync/garden.yml b/services/cron-jobs/shedule-bio-resync/garden.yml new file mode 100644 index 000000000..5385add1f --- /dev/null +++ b/services/cron-jobs/shedule-bio-resync/garden.yml @@ -0,0 +1,43 @@ +kind: Build +name: shedule-bio-resync +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: shedule-bio-resync + SERVICE_PATH: services/cron-jobs/shedule-bio-resync + +--- +kind: Deploy +name: shedule-bio-resync +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.shedule-bio-resync, deploy.mongo] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + +spec: + files: [./manifests/*] + + patchResources: + - name: shedule-bio-resync + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: shedule-bio-resync + image: ${actions.build.shedule-bio-resync.outputs.deploymentImageId} diff --git a/services/cron-jobs/shedule-bio-resync/manifests/ConfigMap.yaml b/services/cron-jobs/shedule-bio-resync/manifests/ConfigMap.yaml new file mode 100644 index 000000000..ff258014e --- /dev/null +++ b/services/cron-jobs/shedule-bio-resync/manifests/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: shedule-bio-resync +data: + DB_URL: ${var.DB_URL} diff --git a/services/cron-jobs/shedule-bio-resync/manifests/CronJob.yaml b/services/cron-jobs/shedule-bio-resync/manifests/CronJob.yaml new file mode 100644 index 000000000..3de8643f9 --- /dev/null +++ b/services/cron-jobs/shedule-bio-resync/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: shedule-bio-resync + labels: + app: shedule-bio-resync +spec: + schedule: '55 2 * */1 *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: shedule-bio-resync + image: democracy/shedule-bio-resync:0.1.5 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: shedule-bio-resync diff --git a/services/cron-jobs/shedule-bio-resync/package.json b/services/cron-jobs/shedule-bio-resync/package.json index 4dbb1271f..6429025d6 100644 --- a/services/cron-jobs/shedule-bio-resync/package.json +++ b/services/cron-jobs/shedule-bio-resync/package.json @@ -12,7 +12,7 @@ "start": "node ./build/index.js" }, "dependencies": { - "@democracy-deutschland/democracy-common": "^0.2.7" + "@democracy-deutschland/democracy-common": "workspace:*" }, "devDependencies": { "dotenv": "^10.0.0", diff --git a/services/cron-jobs/sync-deputy-profiles/garden.yml b/services/cron-jobs/sync-deputy-profiles/garden.yml new file mode 100644 index 000000000..5320c9599 --- /dev/null +++ b/services/cron-jobs/sync-deputy-profiles/garden.yml @@ -0,0 +1,44 @@ +kind: Build +name: sync-deputy-profiles +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: sync-deputy-profiles + SERVICE_PATH: services/cron-jobs/sync-deputy-profiles + +--- +kind: Deploy +name: sync-deputy-profiles +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.sync-deputy-profiles, deploy.mongo, deploy.bundestag-io-api] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + BUNDESTAGIO_SERVER_URL: http://bundestag-io-api + +spec: + files: [./manifests/*] + + patchResources: + - name: sync-deputy-profiles + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync-deputy-profiles + image: ${actions.build.sync-deputy-profiles.outputs.deploymentImageId} diff --git a/services/cron-jobs/sync-deputy-profiles/manifests/ConfigMap.yaml b/services/cron-jobs/sync-deputy-profiles/manifests/ConfigMap.yaml new file mode 100644 index 000000000..914020126 --- /dev/null +++ b/services/cron-jobs/sync-deputy-profiles/manifests/ConfigMap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sync-deputy-profiles +data: + DB_URL: ${var.DB_URL} + BUNDESTAGIO_SERVER_URL: ${var.BUNDESTAGIO_SERVER_URL} diff --git a/services/cron-jobs/sync-deputy-profiles/manifests/CronJob.yaml b/services/cron-jobs/sync-deputy-profiles/manifests/CronJob.yaml new file mode 100644 index 000000000..c39630091 --- /dev/null +++ b/services/cron-jobs/sync-deputy-profiles/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: sync-deputy-profiles + labels: + app: sync-deputy-profiles +spec: + schedule: '3-59/10 * * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: sync-deputy-profiles + image: democracy/sync-deputy-profiles:0.1.12 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: sync-deputy-profiles diff --git a/services/cron-jobs/sync-named-polls/garden.yml b/services/cron-jobs/sync-named-polls/garden.yml new file mode 100644 index 000000000..e721a4689 --- /dev/null +++ b/services/cron-jobs/sync-named-polls/garden.yml @@ -0,0 +1,44 @@ +kind: Build +name: sync-named-polls +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: sync-named-polls + SERVICE_PATH: services/cron-jobs/sync-named-polls + +--- +kind: Deploy +name: sync-named-polls +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.sync-named-polls, deploy.mongo, deploy.bundestag-io-api] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + BUNDESTAGIO_SERVER_URL: http://bundestag-io-api + +spec: + files: [./manifests/*] + + patchResources: + - name: sync-named-polls + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync-named-polls + image: ${actions.build.sync-named-polls.outputs.deploymentImageId} diff --git a/services/cron-jobs/sync-named-polls/manifests/ConfigMap.yaml b/services/cron-jobs/sync-named-polls/manifests/ConfigMap.yaml new file mode 100644 index 000000000..f1b263fa7 --- /dev/null +++ b/services/cron-jobs/sync-named-polls/manifests/ConfigMap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sync-named-polls +data: + DB_URL: ${var.DB_URL} + BUNDESTAGIO_SERVER_URL: ${var.BUNDESTAGIO_SERVER_URL} diff --git a/services/cron-jobs/sync-named-polls/manifests/CronJob.yaml b/services/cron-jobs/sync-named-polls/manifests/CronJob.yaml new file mode 100644 index 000000000..39ac5f431 --- /dev/null +++ b/services/cron-jobs/sync-named-polls/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: sync-named-polls + labels: + app: sync-named-polls +spec: + schedule: '5-59/10 * * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: sync-named-polls + image: democracy/sync-named-polls:0.1.14 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: sync-named-polls diff --git a/services/cron-jobs/sync-procedures/garden.yml b/services/cron-jobs/sync-procedures/garden.yml new file mode 100644 index 000000000..5a05f4b10 --- /dev/null +++ b/services/cron-jobs/sync-procedures/garden.yml @@ -0,0 +1,44 @@ +kind: Build +name: sync-procedures +description: Import procedures for the bundestag.io API +type: container +source: + path: ../../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: sync-procedures + SERVICE_PATH: services/cron-jobs/sync-procedures + +--- +kind: Deploy +name: sync-procedures +type: kubernetes +description: Deploy the importer of procedures for the bundestag.io API +dependencies: [build.sync-procedures, deploy.mongo, deploy.bundestag-io-api] + +disabled: ${!(environment.name == "prod" || environment.name == "local-prod")} + +variables: + DB_URL: mongodb://democracy-mongo:27017/democracy + BUNDESTAGIO_SERVER_URL: http://bundestag-io-api + +spec: + files: [./manifests/*] + + patchResources: + - name: sync-procedures + kind: CronJob + patch: + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync-procedures + image: ${actions.build.sync-procedures.outputs.deploymentImageId} diff --git a/services/cron-jobs/sync-procedures/manifests/ConfigMap.yaml b/services/cron-jobs/sync-procedures/manifests/ConfigMap.yaml new file mode 100644 index 000000000..fcf8550c4 --- /dev/null +++ b/services/cron-jobs/sync-procedures/manifests/ConfigMap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sync-procedures +data: + DB_URL: ${var.DB_URL} + BUNDESTAGIO_SERVER_URL: ${var.BUNDESTAGIO_SERVER_URL} diff --git a/services/cron-jobs/sync-procedures/manifests/CronJob.yaml b/services/cron-jobs/sync-procedures/manifests/CronJob.yaml new file mode 100644 index 000000000..596a6286a --- /dev/null +++ b/services/cron-jobs/sync-procedures/manifests/CronJob.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: sync-procedures + labels: + app: sync-procedures +spec: + schedule: '0 0/2 * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 200 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 86400 + containers: + - name: sync-procedures + image: democracy/sync-procedures:0.1.19 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: sync-procedures diff --git a/services/qr-code-handler/garden.yml b/services/qr-code-handler/garden.yml new file mode 100644 index 000000000..5e661a246 --- /dev/null +++ b/services/qr-code-handler/garden.yml @@ -0,0 +1,39 @@ +kind: Build +name: qr-code-handler +type: container +source: + path: ../../ +include: + - ./**/* + - ./infra/Dockerfile.service +spec: + dockerfile: ./infra/Dockerfile.service + buildArgs: + NODE_VERSION: 18.18.2 + SERVICE: qr-code-handler + SERVICE_PATH: services/qr-code-handler + +--- +kind: Deploy +name: qr-code-handler +type: kubernetes +dependencies: [build.qr-code-handler, deploy.mongo] + +spec: + files: [./manifests/*] # <--- Tell Garden what manifests to use + + defaultTarget: # <--- This tells Garden what "target" to use for logs, code syncing and more + kind: Deployment + name: qr-code-handler + + # Patch the K8s manifests for the api service so that we can set the correct image + patchResources: + - name: qr-code-handler + kind: Deployment + patch: + spec: + template: + spec: + containers: + - name: qr-code-handler + image: ${actions.build.qr-code-handler.outputs.deploymentImageId} # <--- Reference the output from the Build action diff --git a/services/qr-code-handler/manifests/deployment.yaml b/services/qr-code-handler/manifests/deployment.yaml new file mode 100644 index 000000000..35a2e2fa3 --- /dev/null +++ b/services/qr-code-handler/manifests/deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: qr-code-handler + labels: + app: qr-code-handler +spec: + replicas: 1 + selector: + matchLabels: + app: qr-code-handler + template: + metadata: + labels: + app: qr-code-handler + spec: + containers: + - name: qr-code-handler + image: democracy/qr-code-handler:1.0.9 + imagePullPolicy: IfNotPresent + env: + - name: PORT + value: '3000' + - name: DB_URL + value: mongodb://democracy-mongo:27017/qr-code + ports: + - containerPort: 3000 + name: http + protocol: TCP + resources: + limits: + cpu: '1' + memory: 1Gi + requests: + cpu: 10m + memory: 90Mi diff --git a/services/qr-code-handler/manifests/ingress.yaml b/services/qr-code-handler/manifests/ingress.yaml new file mode 100644 index 000000000..2fccffe48 --- /dev/null +++ b/services/qr-code-handler/manifests/ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: qr-code-handler + labels: + app: qr-code-handler +spec: + ingressClassName: nginx + rules: + - host: 'qr-code-handler.${var.hostname}' + http: + paths: + - backend: + service: + name: qr-code-handler + port: + number: 80 + path: / + pathType: Prefix diff --git a/services/qr-code-handler/manifests/service.yaml b/services/qr-code-handler/manifests/service.yaml new file mode 100644 index 000000000..c77dbd089 --- /dev/null +++ b/services/qr-code-handler/manifests/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: qr-code-handler + name: qr-code-handler +spec: + type: ClusterIP + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 3000 + selector: + app: qr-code-handler diff --git a/services/qr-code-handler/src/index.ts b/services/qr-code-handler/src/index.ts index a87818dfc..20b05b7d4 100644 --- a/services/qr-code-handler/src/index.ts +++ b/services/qr-code-handler/src/index.ts @@ -1,12 +1,12 @@ -import express from "express"; -import mongoConnect from "./mongoose"; -import { EntryModel } from "./models"; +import express from 'express'; +import mongoConnect from './mongoose'; +import { EntryModel } from './models'; const app = express(); const port = 3000; -app.get("/:code", (req, _res, next) => { - var code = req.params.code?.toLowerCase(); +app.get('/:code', (req, _res, next) => { + const code = req.params.code?.toLowerCase(); if (code && code.length > 0 && code.match(/^[0-9a-z]+$/)) { console.log(`CODE: "${code}" redirected`); EntryModel.create({ code }); @@ -17,8 +17,8 @@ app.get("/:code", (req, _res, next) => { next(); }); -app.get("*", (_req, res) => { - res.status(301).redirect("https://www.democracy-deutschland.de"); +app.get('*', (_req, res) => { + res.status(301).redirect('https://www.democracy-deutschland.de'); }); app.listen(port, async () => {