Merge branch 'main' into 144-add-macros-to-reduce-boilerplate

.github/workflows/conventional-commits.yml

@@ -0,0 +1,26 @@
+name: Conventional commits
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+
+jobs:
+  lint_pr_title:
+    name: Lint PR title
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Install commitlint
+        run: |
+          npm install @commitlint/config-conventional
+          npm install commitlint@latest
+      - name: Lint title
+        run: npx --no -- commitlint <<< "${{ github.event.pull_request.title }}"

.github/workflows/release.yml

@@ -0,0 +1,25 @@
+name: Release
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Install Semantic Release and plugins
+        run: npm install -g semantic-release @semantic-release/git @semantic-release/changelog @semantic-release/github @semantic-release/commit-analyzer @semantic-release/release-notes-generator
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_ADMIN_TOKEN }}
+        run: npx semantic-release

.releaserc

@@ -0,0 +1,22 @@
+{
+  "branches": ["main"],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+      [
+        "@semantic-release/changelog",
+        {
+          "changelogFile": "CHANGELOG.md"
+        }
+      ],
+      [
+        "@semantic-release/git",
+        {
+          "assets": [
+            "CHANGELOG.md"
+          ]
+        }
+      ],
+      "@semantic-release/github"
+  ]
+}

CHANGELOG.md

@@ -0,0 +1,12 @@
+# 1.0.0 (2024-08-27)
+
+
+### Bug Fixes
+
+* **#145:** docker compose instead of docker-compose ([b4ecea6](https://github.com/medic/cht-pipeline/commit/b4ecea6ac3abddd2d49e66dbd30a87409eae91dd)), closes [#145](https://github.com/medic/cht-pipeline/issues/145)
+
+
+### Features
+
+* **#148:** add automatic releases and versioning ([#152](https://github.com/medic/cht-pipeline/issues/152)) ([52cf12a](https://github.com/medic/cht-pipeline/commit/52cf12a50083a1f343a138cb4c5e1e5166644d7a)), closes [#148](https://github.com/medic/cht-pipeline/issues/148)
+* **#72:** Replace hardcoded values with env variables ([#73](https://github.com/medic/cht-pipeline/issues/73)) ([3dfa8e7](https://github.com/medic/cht-pipeline/commit/3dfa8e710ae45531f999788f17124a78d69d46d0)), closes [#72](https://github.com/medic/cht-pipeline/issues/72)

README.md

@@ -19,3 +19,19 @@ Follow the instructions in [the Local CHT Sync Setup documentation](https://docs
 # set environment variables, install dbt dependencies, seed data, run dbt, run test
 ./run_dbt_tests.sh
 ```
+
+## Release Process
+This repo has an automated release process where each feature/bug fix will be released immediately after it is merged to `main`. The release type is determined by the commit message format. Have a look at the development workflow in the [Contributor Handbook](https://docs.communityhealthtoolkit.org/contribute/code/workflow/) for more information.
+
+### Commit message format
+
+The commit format should follow the convention outlined in the [CHT docs](https://docs.communityhealthtoolkit.org/contribute/code/workflow/#commit-message-format).
+Examples are provided below.
+
+| Type        | Example commit message                                                                              | Release type |
+|-------------|-----------------------------------------------------------------------------------------------------|--------------|
+| Bug fixes   | fix(#123): rename column names                                                                      | patch        |
+| Performance | perf(#789): add new indexes                                                                         | patch        |
+| Features    | feat(#456): add new model                                                                           | minor        |
+| Non-code    | chore(#123): update README                                                                          | none         |
+| Breaking    | perf(#2): remove data_record model <br/> BREAKING CHANGE: form models should now read from new_model| major        |

commitlint.config.js

@@ -0,0 +1,2 @@
+module.exports = { extends: ['@commitlint/config-conventional'] };
+

models/contacts/contact.sql

@@ -28,7 +28,7 @@ SELECT
   NULLIF(doc->> 'muted', '') AS muted
 FROM {{ ref('document_metadata') }} document_metadata
 INNER JOIN
-  {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} source_table
+  {{ source('couchdb', env_var('POSTGRES_TABLE')) }} source_table
   ON source_table._id = document_metadata.uuid
 WHERE
   document_metadata.doc_type IN ('contact', 'clinic', 'district_hospital', 'health_center', 'person')

models/contacts/contacts.yml

@@ -12,21 +12,30 @@ models:
           - type: foreign_key
             expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE"
           - type: unique
-        tests:
+        data_tests:
           - not_null
+          - unique
           - relationships:
               to: ref('document_metadata')
               field: uuid
       - name: saved_timestamp
         data_type: timestamp
+        data_tests:
+          - not_null
       - name: reported
         data_type: timestamp with time zone
+        data_tests:
+          - not_null
       - name: parent_uuid
         data_type: string
       - name: name
         data_type: string
+        data_tests:
+          - not_null
       - name: contact_type
         data_type: string
+        data_tests:
+          - not_null
       - name: phone
         data_type: string
       - name: phone2
@@ -37,8 +46,6 @@ models:
         data_type: string
       - name: contact_id
         data_type: string
-      - name: contact_id
-        data_type: string
       - name: muted
         data_type: string
   - name: person
@@ -51,8 +58,9 @@ models:
         constraints:
           - type: foreign_key
             expression: "{{ env_var('POSTGRES_SCHEMA') }}.contact (uuid) ON DELETE CASCADE"
-        tests:
+        data_tests:
           - not_null
+          - unique
           - relationships:
               to: ref('contact')
               field: uuid
@@ -72,7 +80,7 @@ models:
         constraints:
           - type: foreign_key
             expression: "{{ env_var('POSTGRES_SCHEMA') }}.contact (uuid) ON DELETE CASCADE"
-        tests:
+        data_tests:
           - not_null
           - relationships:
               to: ref('contact')
@@ -91,7 +99,7 @@ models:
         constraints:
           - type: foreign_key
             expression: "{{ env_var('POSTGRES_SCHEMA') }}.contact (uuid) ON DELETE CASCADE"
-        tests:
+        data_tests:
           - not_null
           - relationships:
               to: ref('contact')

models/contacts/patient.sql

@@ -16,7 +16,7 @@ SELECT
   person.saved_timestamp,
   couchdb.doc->>'patient_id' as patient_id
 FROM {{ ref('person') }} person
-INNER JOIN {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} couchdb ON couchdb._id = uuid
+INNER JOIN {{ source('couchdb', env_var('POSTGRES_TABLE')) }} couchdb ON couchdb._id = uuid
 WHERE couchdb.doc->>'patient_id' IS NOT NULL
 {% if is_incremental() %}
   AND person.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }}

models/contacts/person.sql

@@ -16,7 +16,7 @@ SELECT
   (couchdb.doc->>'date_of_birth')::date as date_of_birth,
   couchdb.doc->>'sex' as sex
 FROM {{ ref("contact") }} contact
-INNER JOIN {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} couchdb ON couchdb._id = uuid
+INNER JOIN {{ source('couchdb', env_var('POSTGRES_TABLE')) }} couchdb ON couchdb._id = uuid
 WHERE contact.contact_type = 'person'
 {% if is_incremental() %}
   AND contact.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }}

models/contacts/place.sql

@@ -16,7 +16,7 @@ SELECT
   contact.saved_timestamp,
   couchdb.doc->>'place_id' as place_id
 FROM {{ ref('contact') }} contact
-INNER JOIN {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} couchdb ON couchdb._id = uuid
+INNER JOIN {{ source('couchdb', env_var('POSTGRES_TABLE')) }} couchdb ON couchdb._id = uuid
 WHERE
   (
     (couchdb.doc->>'place_id' IS NOT NULL) OR

models/contacts/tests/contacts.yml

@@ -0,0 +1,20 @@
+unit_tests:
+  - name: test_contact_model_transformation_and_data_integrity
+    description: |
+      This unit test validates the transformation logic in the `contact` model and ensures data integrity.
+      It uses fixture data for both `document_metadata` and `source_table` to test the complete logic.
+    model: contact
+    overrides:
+      macros:
+        is_incremental: false
+    given:
+      - input: ref('document_metadata')
+        format: csv
+        fixture: contact_document_metadata_initial
+      - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}")
+        format: csv
+        fixture: contact_source_table_initial
+    expect:
+      format: csv
+      fixture: contact_initial_expected
+

models/contacts/tests/patients.yml

@@ -0,0 +1,19 @@
+unit_tests:
+  - name: test_patient_model_transformation_and_data_integrity
+    description: |
+      This unit test validates the transformation logic in the `patient` model and ensures data integrity.
+      It uses fixture data for both `person` and `couchdb` to test the complete logic.
+    model: patient
+    overrides:
+      macros:
+        is_incremental: false
+    given:
+      - input: ref('person')
+        format: csv
+        fixture: patient_person_initial
+      - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}")
+        format: csv
+        fixture: patient_source_table_initial
+    expect:
+      format: csv
+      fixture: patient_initial_expected

models/contacts/tests/persons.yml

@@ -0,0 +1,19 @@
+unit_tests:
+  - name: test_person_model_transformation_and_data_integrity
+    description: |
+      This unit test validates the transformation logic in the `person` model and ensures data integrity.
+      It uses fixture data for `contact` and `couchdb` to test the complete logic.
+    model: person
+    overrides:
+      macros:
+        is_incremental: false
+    given:
+      - input: ref('contact')
+        format: csv
+        fixture: person_contact_initial
+      - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}")
+        format: csv
+        fixture: person_source_table_initial
+    expect:
+      format: csv
+      fixture: person_initial_expected

models/contacts/tests/places.yml

@@ -0,0 +1,19 @@
+unit_tests:
+  - name: test_place_model_transformation_and_data_integrity
+    description: |
+      This unit test validates the transformation logic in the `place` model and ensures data integrity.
+      It uses fixture data for both `contact` and `couchdb` to test the complete logic.
+    model: place
+    overrides:
+      macros:
+        is_incremental: false
+    given:
+      - input: ref('contact')
+        format: csv
+        fixture: place_contact_initial
+      - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}")
+        format: csv
+        fixture: place_source_table_initial
+    expect:
+      format: csv
+      fixture: place_initial_expected

models/forms/data_record.sql

@@ -40,7 +40,7 @@ SELECT
   doc->'contact'->'parent'->'parent'->>'_id' as grandparent_uuid
 FROM {{ ref('document_metadata') }} document_metadata
 INNER JOIN
-  {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} source_table
+  {{ source('couchdb', env_var('POSTGRES_TABLE')) }} source_table
   ON source_table._id = document_metadata.uuid
 WHERE
   document_metadata.doc_type = 'data_record'

models/forms/forms.yml

@@ -12,7 +12,7 @@ models:
           - type: foreign_key
             expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE"
           - type: unique
-        tests:
+        data_tests:
           - not_null
           - relationships:
               to: ref('document_metadata')

models/forms/tests/forms.yml

@@ -0,0 +1,21 @@
+version: 2
+
+unit_tests:
+  - name: test_data_record_model_transformation_and_data_integrity
+    description: |
+      This unit test validates the transformation logic in the `data_record` model and ensures data integrity.
+      It uses fixture data for both `document_metadata` and `couchdb` to test the complete logic.
+    model: data_record
+    overrides:
+      macros:
+        is_incremental: false
+    given:
+      - input: ref('document_metadata')
+        format: csv
+        fixture: data_record_document_metadata_initial
+      - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}")
+        format: csv
+        fixture: data_record_source_table_initial
+    expect:
+      format: csv
+      fixture: data_record_initial_expected

models/root/document_metadata.sql

@@ -18,7 +18,7 @@ SELECT
   _deleted,
   saved_timestamp,
   doc->>'type' as doc_type
-FROM {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} source_table
+from {{ source('couchdb', env_var('POSTGRES_TABLE')) }} source_table
 {% if is_incremental() %}
 WHERE source_table.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }}
 {% endif %}

models/root/root.yml

@@ -16,11 +16,20 @@ models:
         data_type: string
         constraints:
           - type: unique
-        tests:
+        data-tests:
           - not_null
+          - unique
       - name: saved_timestamp
         data_type: timestamp
+        data-tests:
+          - not_null
       - name: _deleted
         data_type: boolean
+        data-tests:
+          - not_null
+          - accepted_values:
+              values: [true, false]
       - name: doc_type
         data_type: string
+        data-tests:
+          - not_null

models/root/tests/document_metadata.yml

@@ -0,0 +1,14 @@
+unit_tests:
+  - name: test_document_metadata
+    model: document_metadata
+    overrides:
+      macros:
+        is_incremental: false
+    given:
+      - input: source('couchdb', "{{ env_var('POSTGRES_TABLE') }}")
+        format: csv
+        fixture: document_metadata_initial
+    expect:
+      format: csv
+      fixture: document_metadata_initial_expected
+