diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..2e27f2e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,36 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: 'Type: Bug' +assignees: '' + +--- + + + +**Describe the bug** + + +**To Reproduce** + +1. Step 1 +2. Step 2 + +**Expected behavior** + + +**Logs** + + +**Screenshots** + + +**Environment** +- Instance +- Client platform: (eg: Windows, MacOS, Linux) + +**Additional context** + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..b92c70c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,2 @@ +blank_issues_enabled: true + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..4a539a6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: 'Type: Feature' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** + + +**Describe the solution you'd like** + + +**Describe alternatives you've considered** + + +**Additional context** + diff --git a/.github/ISSUE_TEMPLATE/improvement.md b/.github/ISSUE_TEMPLATE/improvement.md new file mode 100644 index 0000000..ecdbe88 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/improvement.md @@ -0,0 +1,20 @@ +--- +name: Improvement +about: Suggest something to make an existing feature better +title: '' +labels: 'Type: Improvement' +assignees: '' + +--- + +**What feature do you want to improve?** + + +**Describe the improvement you'd like** + + +**Describe alternatives you've considered** + + +**Additional context** + diff --git a/.github/ISSUE_TEMPLATE/technical_issue.md b/.github/ISSUE_TEMPLATE/technical_issue.md new file mode 100644 index 0000000..6204092 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/technical_issue.md @@ -0,0 +1,17 @@ +--- +name: Technical issue +about: Suggest an improvement users won't notice +title: '' +labels: 'Type: Technical issue' +assignees: '' + +--- + +**Describe the issue** + + +**Describe the improvement you'd like** + + +**Describe alternatives you've considered** + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 28bc774..8d08dea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,9 +1,10 @@ name: Build cht-sync containers on: - push: - branches: - - 'main' + workflow_run: + workflows: ["release"] + types: + - completed jobs: build: @@ -18,13 +19,24 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Get the latest release tag + id: get_latest_release + uses: actions/github-script@v6 + with: + script: | + const latestRelease = await github.repos.getLatestRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + return latestRelease.data.tag_name; + - name: Build and push couch2pg uses: docker/build-push-action@v4 with: context: ./couch2pg file: ./couch2pg/Dockerfile push: true - tags: medicmobile/cht-sync-couch2pg:latest + tags: medicmobile/cht-sync-couch2pg:${{ steps.get_latest_release.outputs.result }} - name: Build and push dataemon uses: docker/build-push-action@v4 @@ -32,4 +44,4 @@ jobs: context: ./dbt file: ./dbt/Dockerfile push: true - tags: medicmobile/dataemon:latest + tags: medicmobile/dataemon:${{ steps.get_latest_release.outputs.result }} diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml new file mode 100644 index 0000000..e0c5c39 --- /dev/null +++ b/.github/workflows/conventional-commits.yml @@ -0,0 +1,26 @@ +name: Conventional commits +on: + pull_request: + types: + - opened + - edited + - synchronize + - reopened + +jobs: + lint_pr_title: + name: Lint PR title + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Install commitlint + run: | + npm install @commitlint/config-conventional + npm install commitlint@latest + - name: Lint title + run: npx --no -- commitlint <<< "${{ github.event.pull_request.title }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..aefb82d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,25 @@ +name: Release +on: + push: + branches: + - main + +jobs: + release: + name: Release + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Install Semantic Release and plugins + run: npm install -g semantic-release @semantic-release/git @semantic-release/changelog @semantic-release/github @semantic-release/commit-analyzer @semantic-release/release-notes-generator + - name: Release + env: + GITHUB_TOKEN: ${{ secrets.GH_ADMIN_TOKEN }} + run: npx semantic-release diff --git a/.releaserc b/.releaserc new file mode 100644 index 0000000..39bbe6f --- /dev/null +++ b/.releaserc @@ -0,0 +1,22 @@ +{ + "branches": ["main"], + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + [ + "@semantic-release/changelog", + { + "changelogFile": "CHANGELOG.md" + } + ], + [ + "@semantic-release/git", + { + "assets": [ + "CHANGELOG.md" + ] + } + ], + "@semantic-release/github" + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..cc51ee0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +# 1.0.0 (2024-09-10) + + +### Bug Fixes + +* Change env variables according to cht pipeline updates ([#71](https://github.com/medic/cht-sync/issues/71)) ([c89aadf](https://github.com/medic/cht-sync/commit/c89aadf71c4562dcd1ef79747f4ebc7733796459)) +* Fix numbering ([#50](https://github.com/medic/cht-sync/issues/50)) ([5c93300](https://github.com/medic/cht-sync/commit/5c93300d2009c37b50088e4892795b7bce88a6c2)) + + +### Features + +* **#107:** Adds multi-db watcher support ([#113](https://github.com/medic/cht-sync/issues/113)) ([279d8f2](https://github.com/medic/cht-sync/commit/279d8f25e051c9d3ff9e8b46baa7f5faaecb2290)), closes [#107](https://github.com/medic/cht-sync/issues/107) [#107](https://github.com/medic/cht-sync/issues/107) +* **#112:** drop support for multiple copies of every document ([#115](https://github.com/medic/cht-sync/issues/115)) ([b46f288](https://github.com/medic/cht-sync/commit/b46f288f5cc15b28196f938141df36520d0f4674)), closes [#112](https://github.com/medic/cht-sync/issues/112) [#118](https://github.com/medic/cht-sync/issues/118) +* **#129:** add back automatic pipeline updates ([#130](https://github.com/medic/cht-sync/issues/130)) ([fc73fd7](https://github.com/medic/cht-sync/commit/fc73fd707cd6db76b12d7b03d356709bc726db07)), closes [#129](https://github.com/medic/cht-sync/issues/129) [#129](https://github.com/medic/cht-sync/issues/129) [#129](https://github.com/medic/cht-sync/issues/129) +* **#1:** first release ([ff0fedd](https://github.com/medic/cht-sync/commit/ff0feddeb35f7b78745bd40b7d2fe20e8f99d8c7)), closes [#1](https://github.com/medic/cht-sync/issues/1) +* **#25:** custom databases ([#33](https://github.com/medic/cht-sync/issues/33)) ([cd10db0](https://github.com/medic/cht-sync/commit/cd10db07ad2a0e2879e1eddb6be47c9ba9af10b8)), closes [#25](https://github.com/medic/cht-sync/issues/25) +* **#78:** full refresh on changed objects, only incremental runs continously ([0869ee9](https://github.com/medic/cht-sync/commit/0869ee9a6d4bd7bb4ee07022a55aef09ec085ce3)), closes [#78](https://github.com/medic/cht-sync/issues/78) +* add versioning and releases ([a528aba](https://github.com/medic/cht-sync/commit/a528aba64f3040d8163f2ea5d72f3457acf5dfa0)) +* bind sequence token path to host for persistence ([#88](https://github.com/medic/cht-sync/issues/88)) ([e1c3953](https://github.com/medic/cht-sync/commit/e1c39536fc445aa6f88617bf852f8f41b0fc724f)) +* remove superset container and update Readme ([#64](https://github.com/medic/cht-sync/issues/64)) ([8acbc93](https://github.com/medic/cht-sync/commit/8acbc9384cbc59c1776727ced63dd603d1fd09c7)) +* update logstash base image version and update default configuration files ([#61](https://github.com/medic/cht-sync/issues/61)) ([674582d](https://github.com/medic/cht-sync/commit/674582d08c0b32542b366ca0c46cc03352845ece)) +* update postgres version to 16 ([8bf1e84](https://github.com/medic/cht-sync/commit/8bf1e843b8c4821d460a63cc866d02baad7498bf)) diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000..304c404 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,5 @@ +export default { + extends: [ + '@commitlint/config-conventional' + ] +}; diff --git a/deploy/cht_sync/templates/postgres.yaml b/deploy/cht_sync/templates/postgres.yaml index 99e63c3..74bf556 100644 --- a/deploy/cht_sync/templates/postgres.yaml +++ b/deploy/cht_sync/templates/postgres.yaml @@ -21,7 +21,7 @@ spec: automountServiceAccountToken: false containers: - name: postgres-c - image: postgres:13 + image: postgres:16 resources: limits: cpu: {{ (.Values.postgres).cpu_limit | default "1000m" }} diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml index cff8e22..54d0aed 100644 --- a/docker-compose.postgres.yml +++ b/docker-compose.postgres.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:13 + image: postgres:16 restart: always ports: - 5432:5432 @@ -9,9 +9,6 @@ services: environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - DBT_POSTGRES_USER=${DBT_POSTGRES_USER} - - DBT_POSTGRES_PASSWORD=${DBT_POSTGRES_PASSWORD} - - DBT_POSTGRES_SCHEMA=${DBT_POSTGRES_SCHEMA} - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_TABLES=${COUCHDB_DBS} - POSTGRES_SCHEMA=${POSTGRES_SCHEMA} diff --git a/docker-compose.yml b/docker-compose.yml index 589b26e..b42834e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,12 +27,12 @@ services: build: ./dbt/ working_dir: /dbt/ environment: - - POSTGRES_HOST=${DBT_POSTGRES_HOST} - - POSTGRES_USER=${DBT_POSTGRES_USER} - - POSTGRES_PASSWORD=${DBT_POSTGRES_PASSWORD} + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_TABLE=${POSTGRES_TABLE} - - POSTGRES_SCHEMA=${DBT_POSTGRES_SCHEMA} + - POSTGRES_SCHEMA=${POSTGRES_SCHEMA} - ROOT_POSTGRES_SCHEMA=${POSTGRES_SCHEMA} - CHT_PIPELINE_BRANCH_URL=${CHT_PIPELINE_BRANCH_URL} - DATAEMON_INTERVAL=${DATAEMON_INTERVAL} diff --git a/env.template b/env.template index db378a2..c7fd8e5 100644 --- a/env.template +++ b/env.template @@ -6,15 +6,11 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres POSTGRES_DB=data POSTGRES_SCHEMA=v1 -POSTGRES_TABLE=medic # for dbt use only +POSTGRES_TABLE=couchdb POSTGRES_HOST=postgres # Your postgres instance IP or endpoint in "prod". POSTGRES_PORT=5432 # dbt -DBT_POSTGRES_USER=postgres -DBT_POSTGRES_PASSWORD=postgres -DBT_POSTGRES_SCHEMA=dbt -DBT_POSTGRES_HOST=postgres # Your postgres instance IP or endpoint in "prod". CHT_PIPELINE_BRANCH_URL="https://github.com/medic/cht-pipeline.git#main" DATAEMON_INTERVAL=5 diff --git a/package.json b/package.json index 1d01b0a..e829428 100644 --- a/package.json +++ b/package.json @@ -36,5 +36,6 @@ "pouchdb-core": "^8.0.1", "sinon": "^18.0.0", "uuid": "^9.0.1" - } + }, + "private": true } diff --git a/postgres/init-dbt-resources.sh b/postgres/init-dbt-resources.sh index 454eb7b..2a7ef29 100755 --- a/postgres/init-dbt-resources.sh +++ b/postgres/init-dbt-resources.sh @@ -6,13 +6,13 @@ export PGPASSWORD=$POSTGRES_PASSWORD echo Setting up defaults psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE USER $DBT_POSTGRES_USER WITH PASSWORD '$DBT_POSTGRES_PASSWORD'; - CREATE SCHEMA IF NOT EXISTS $DBT_POSTGRES_SCHEMA AUTHORIZATION $DBT_POSTGRES_USER; + CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD'; + CREATE SCHEMA IF NOT EXISTS $POSTGRES_SCHEMA AUTHORIZATION $POSTGRES_USER; - GRANT USAGE ON SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER; - GRANT SELECT ON ALL TABLES IN SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA $POSTGRES_SCHEMA TO $DBT_POSTGRES_USER; + GRANT USAGE ON SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER; + GRANT SELECT ON ALL TABLES IN SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA $POSTGRES_SCHEMA TO $POSTGRES_USER; - CREATE SCHEMA IF NOT EXISTS $DBT_POSTGRES_SCHEMA AUTHORIZATION $DBT_POSTGRES_USER; + CREATE SCHEMA IF NOT EXISTS $POSTGRES_SCHEMA AUTHORIZATION $POSTGRES_USER; EOSQL