Merge branch 'main' into mrjones-plip-patch-1

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,36 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'Type: Bug'
+assignees: ''
+
+---
+
+<!--
+**Important**: This is a public repository. Anyone in the world can see what's posted here. If you are posting screenshots or log files, please **carefully examine them for** the presence of any kind of **protected health information** (PHI). Images or logs containing PHI _must_ be posted in fully-redacted form, with no visible PHI.
+-->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior. -->
+1. Step 1
+2. Step 2
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Logs**
+<!-- If applicable, include the server logs, and/or browser logs. -->
+
+**Screenshots**
+<!-- If applicable, add screenshots to help explain your problem. -->
+
+**Environment**
+- Instance
+- Client platform: (eg: Windows, MacOS, Linux)
+
+**Additional context**
+<!-- Add any other context about the problem here. What have you tried? Is there a workaround? -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+blank_issues_enabled: true
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: 'Type: Feature'
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+
+**Describe the solution you'd like**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/improvement.md

@@ -0,0 +1,20 @@
+---
+name: Improvement
+about: Suggest something to make an existing feature better
+title: ''
+labels: 'Type: Improvement'
+assignees: ''
+
+---
+
+**What feature do you want to improve?**
+<!-- A clear and concise description of what the problem is. Ex. It would be better to [...] -->
+
+**Describe the improvement you'd like**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/technical_issue.md

@@ -0,0 +1,17 @@
+---
+name: Technical issue
+about: Suggest an improvement users won't notice
+title: ''
+labels: 'Type: Technical issue'
+assignees: ''
+
+---
+
+**Describe the issue**
+<!-- A clear and concise description of what the problem is. -->
+
+**Describe the improvement you'd like**
+<!-- A clear and concise description of what you want to change. -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->

.github/workflows/build.yml

@@ -1,9 +1,10 @@
 name: Build cht-sync containers
 
 on:
-  push:
-    branches:
-      - 'main'
+  workflow_run:
+    workflows: ["release"]
+    types:
+      - completed
 
 jobs:
   build:
@@ -18,18 +19,29 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Get the latest release tag
+        id: get_latest_release
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const latestRelease = await github.repos.getLatestRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            });
+            return latestRelease.data.tag_name;
+
       - name: Build and push couch2pg
         uses: docker/build-push-action@v4
         with:
           context: ./couch2pg
           file: ./couch2pg/Dockerfile
           push: true
-          tags: medicmobile/cht-sync-couch2pg:latest
+          tags: medicmobile/cht-sync-couch2pg:${{ steps.get_latest_release.outputs.result }}
 
       - name: Build and push dataemon
         uses: docker/build-push-action@v4
         with:
           context: ./dbt
           file: ./dbt/Dockerfile
           push: true
-          tags: medicmobile/dataemon:latest
+          tags: medicmobile/dataemon:${{ steps.get_latest_release.outputs.result }}

.github/workflows/conventional-commits.yml

@@ -0,0 +1,26 @@
+name: Conventional commits
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+
+jobs:
+  lint_pr_title:
+    name: Lint PR title
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Install commitlint
+        run: |
+          npm install @commitlint/config-conventional
+          npm install commitlint@latest
+      - name: Lint title
+        run: npx --no -- commitlint <<< "${{ github.event.pull_request.title }}"

.github/workflows/release.yml

@@ -0,0 +1,25 @@
+name: Release
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Install Semantic Release and plugins
+        run: npm install -g semantic-release @semantic-release/git @semantic-release/changelog @semantic-release/github @semantic-release/commit-analyzer @semantic-release/release-notes-generator
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_ADMIN_TOKEN }}
+        run: npx semantic-release

.releaserc

@@ -0,0 +1,22 @@
+{
+  "branches": ["main"],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+      [
+        "@semantic-release/changelog",
+        {
+          "changelogFile": "CHANGELOG.md"
+        }
+      ],
+      [
+        "@semantic-release/git",
+        {
+          "assets": [
+            "CHANGELOG.md"
+          ]
+        }
+      ],
+      "@semantic-release/github"
+  ]
+}

CHANGELOG.md

@@ -0,0 +1,22 @@
+# 1.0.0 (2024-09-10)
+
+
+### Bug Fixes
+
+* Change env variables according to cht pipeline updates ([#71](https://github.com/medic/cht-sync/issues/71)) ([c89aadf](https://github.com/medic/cht-sync/commit/c89aadf71c4562dcd1ef79747f4ebc7733796459))
+* Fix numbering ([#50](https://github.com/medic/cht-sync/issues/50)) ([5c93300](https://github.com/medic/cht-sync/commit/5c93300d2009c37b50088e4892795b7bce88a6c2))
+
+
+### Features
+
+* **#107:** Adds multi-db watcher support ([#113](https://github.com/medic/cht-sync/issues/113)) ([279d8f2](https://github.com/medic/cht-sync/commit/279d8f25e051c9d3ff9e8b46baa7f5faaecb2290)), closes [#107](https://github.com/medic/cht-sync/issues/107) [#107](https://github.com/medic/cht-sync/issues/107)
+* **#112:** drop support for multiple copies of every document ([#115](https://github.com/medic/cht-sync/issues/115)) ([b46f288](https://github.com/medic/cht-sync/commit/b46f288f5cc15b28196f938141df36520d0f4674)), closes [#112](https://github.com/medic/cht-sync/issues/112) [#118](https://github.com/medic/cht-sync/issues/118)
+* **#129:** add back automatic pipeline updates ([#130](https://github.com/medic/cht-sync/issues/130)) ([fc73fd7](https://github.com/medic/cht-sync/commit/fc73fd707cd6db76b12d7b03d356709bc726db07)), closes [#129](https://github.com/medic/cht-sync/issues/129) [#129](https://github.com/medic/cht-sync/issues/129) [#129](https://github.com/medic/cht-sync/issues/129)
+* **#1:** first release ([ff0fedd](https://github.com/medic/cht-sync/commit/ff0feddeb35f7b78745bd40b7d2fe20e8f99d8c7)), closes [#1](https://github.com/medic/cht-sync/issues/1)
+* **#25:** custom databases ([#33](https://github.com/medic/cht-sync/issues/33)) ([cd10db0](https://github.com/medic/cht-sync/commit/cd10db07ad2a0e2879e1eddb6be47c9ba9af10b8)), closes [#25](https://github.com/medic/cht-sync/issues/25)
+* **#78:** full refresh on changed objects, only incremental runs continously ([0869ee9](https://github.com/medic/cht-sync/commit/0869ee9a6d4bd7bb4ee07022a55aef09ec085ce3)), closes [#78](https://github.com/medic/cht-sync/issues/78)
+* add versioning and releases ([a528aba](https://github.com/medic/cht-sync/commit/a528aba64f3040d8163f2ea5d72f3457acf5dfa0))
+* bind sequence token path to host for persistence ([#88](https://github.com/medic/cht-sync/issues/88)) ([e1c3953](https://github.com/medic/cht-sync/commit/e1c39536fc445aa6f88617bf852f8f41b0fc724f))
+* remove superset container and update Readme ([#64](https://github.com/medic/cht-sync/issues/64)) ([8acbc93](https://github.com/medic/cht-sync/commit/8acbc9384cbc59c1776727ced63dd603d1fd09c7))
+* update logstash base image version and update default configuration files ([#61](https://github.com/medic/cht-sync/issues/61)) ([674582d](https://github.com/medic/cht-sync/commit/674582d08c0b32542b366ca0c46cc03352845ece))
+* update postgres version to 16 ([8bf1e84](https://github.com/medic/cht-sync/commit/8bf1e843b8c4821d460a63cc866d02baad7498bf))

commitlint.config.js

@@ -0,0 +1,5 @@
+export default {
+  extends: [
+    '@commitlint/config-conventional'
+  ]
+};

deploy/cht_sync/templates/postgres.yaml

@@ -21,7 +21,7 @@ spec:
       automountServiceAccountToken: false
       containers:
         - name: postgres-c
-          image: postgres:13
+          image: postgres:16
           resources:
             limits:
               cpu: {{ (.Values.postgres).cpu_limit | default "1000m" }}

docker-compose.postgres.yml

@@ -1,6 +1,6 @@
 services:
   postgres:
-    image: postgres:13
+    image: postgres:16
     restart: always
     ports:
       - 5432:5432
@@ -9,9 +9,6 @@ services:
     environment:
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - DBT_POSTGRES_USER=${DBT_POSTGRES_USER}
-      - DBT_POSTGRES_PASSWORD=${DBT_POSTGRES_PASSWORD}
-      - DBT_POSTGRES_SCHEMA=${DBT_POSTGRES_SCHEMA}
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_TABLES=${COUCHDB_DBS}
       - POSTGRES_SCHEMA=${POSTGRES_SCHEMA}

docker-compose.yml

@@ -27,12 +27,12 @@ services:
     build: ./dbt/
     working_dir: /dbt/
     environment:
-      - POSTGRES_HOST=${DBT_POSTGRES_HOST}
-      - POSTGRES_USER=${DBT_POSTGRES_USER}
-      - POSTGRES_PASSWORD=${DBT_POSTGRES_PASSWORD}
+      - POSTGRES_HOST=${POSTGRES_HOST}
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_TABLE=${POSTGRES_TABLE}
-      - POSTGRES_SCHEMA=${DBT_POSTGRES_SCHEMA}
+      - POSTGRES_SCHEMA=${POSTGRES_SCHEMA}
       - ROOT_POSTGRES_SCHEMA=${POSTGRES_SCHEMA}
       - CHT_PIPELINE_BRANCH_URL=${CHT_PIPELINE_BRANCH_URL}
       - DATAEMON_INTERVAL=${DATAEMON_INTERVAL}

env.template

@@ -6,15 +6,11 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_DB=data
 POSTGRES_SCHEMA=v1
-POSTGRES_TABLE=medic # for dbt use only
+POSTGRES_TABLE=couchdb 
 POSTGRES_HOST=postgres # Your postgres instance IP or endpoint in "prod".
 POSTGRES_PORT=5432
 
 # dbt
-DBT_POSTGRES_USER=postgres
-DBT_POSTGRES_PASSWORD=postgres
-DBT_POSTGRES_SCHEMA=dbt
-DBT_POSTGRES_HOST=postgres # Your postgres instance IP or endpoint in "prod".
 CHT_PIPELINE_BRANCH_URL="https://github.com/medic/cht-pipeline.git#main"
 DATAEMON_INTERVAL=5
 

package.json

@@ -36,5 +36,6 @@
     "pouchdb-core": "^8.0.1",
     "sinon": "^18.0.0",
     "uuid": "^9.0.1"
-  }
+  },
+  "private": true
 }

postgres/init-dbt-resources.sh

@@ -6,13 +6,13 @@ export PGPASSWORD=$POSTGRES_PASSWORD
 echo Setting up defaults
 
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    CREATE USER $DBT_POSTGRES_USER WITH PASSWORD '$DBT_POSTGRES_PASSWORD';
-    CREATE SCHEMA IF NOT EXISTS $DBT_POSTGRES_SCHEMA AUTHORIZATION $DBT_POSTGRES_USER;
+    CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD';
+    CREATE SCHEMA IF NOT EXISTS $POSTGRES_SCHEMA AUTHORIZATION $POSTGRES_USER;
 
-    GRANT USAGE ON SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER;
-    GRANT SELECT ON ALL TABLES IN SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER;
-    GRANT SELECT ON ALL SEQUENCES IN SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER;
-    GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER;
+    GRANT USAGE ON SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER;
+    GRANT SELECT ON ALL TABLES IN SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER;
+    GRANT SELECT ON ALL SEQUENCES IN SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER;
+    GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER;
 
-    CREATE SCHEMA IF NOT EXISTS $DBT_POSTGRES_SCHEMA AUTHORIZATION $DBT_POSTGRES_USER;
+    CREATE SCHEMA IF NOT EXISTS $POSTGRES_SCHEMA AUTHORIZATION $POSTGRES_USER;
 EOSQL