merging from medic upstream

.dbtignore

@@ -0,0 +1,2 @@
+# .dbtignore
+init.sql

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,36 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'Type: Bug'
+assignees: ''
+
+---
+
+<!--
+**Important**: This is a public repository. Anyone in the world can see what's posted here. If you are posting screenshots or log files, please **carefully examine them for** the presence of any kind of **protected health information** (PHI). Images or logs containing PHI _must_ be posted in fully-redacted form, with no visible PHI.
+-->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior. -->
+1. Step 1
+2. Step 2
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Logs**
+<!-- If applicable, include the server logs, and/or browser logs. -->
+
+**Screenshots**
+<!-- If applicable, add screenshots to help explain your problem. -->
+
+**Environment**
+- Instance
+- Client platform: (eg: Windows, MacOS, Linux)
+
+**Additional context**
+<!-- Add any other context about the problem here. What have you tried? Is there a workaround? -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+blank_issues_enabled: true
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: 'Type: Feature'
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+
+**Describe the solution you'd like**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/improvement.md

@@ -0,0 +1,20 @@
+---
+name: Improvement
+about: Suggest something to make an existing feature better
+title: ''
+labels: 'Type: Improvement'
+assignees: ''
+
+---
+
+**What feature do you want to improve?**
+<!-- A clear and concise description of what the problem is. Ex. It would be better to [...] -->
+
+**Describe the improvement you'd like**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/technical_issue.md

@@ -0,0 +1,17 @@
+---
+name: Technical issue
+about: Suggest an improvement users won't notice
+title: ''
+labels: 'Type: Technical issue'
+assignees: ''
+
+---
+
+**Describe the issue**
+<!-- A clear and concise description of what the problem is. -->
+
+**Describe the improvement you'd like**
+<!-- A clear and concise description of what you want to change. -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->

.github/workflows/conventional-commits.yml

@@ -0,0 +1,26 @@
+name: Conventional commits
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+
+jobs:
+  lint_pr_title:
+    name: Lint PR title
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Install commitlint
+        run: |
+          npm install @commitlint/config-conventional
+          npm install commitlint@latest
+      - name: Lint title
+        run: npx --no -- commitlint <<< "${{ github.event.pull_request.title }}"

.github/workflows/release.yml

@@ -0,0 +1,25 @@
+name: Release
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Install Semantic Release and plugins
+        run: npm install -g semantic-release @semantic-release/git @semantic-release/changelog @semantic-release/github @semantic-release/commit-analyzer @semantic-release/release-notes-generator
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_ADMIN_TOKEN }}
+        run: npx semantic-release

.releaserc

@@ -0,0 +1,22 @@
+{
+  "branches": ["main"],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+      [
+        "@semantic-release/changelog",
+        {
+          "changelogFile": "CHANGELOG.md"
+        }
+      ],
+      [
+        "@semantic-release/git",
+        {
+          "assets": [
+            "CHANGELOG.md"
+          ]
+        }
+      ],
+      "@semantic-release/github"
+  ]
+}

CHANGELOG.md

@@ -0,0 +1,26 @@
+## [1.1.1](https://github.com/medic/cht-pipeline/compare/v1.1.0...v1.1.1) (2024-09-04)
+
+
+### Bug Fixes
+
+* **#156:** unit test fixture not found when integrating with cht-sync ([#158](https://github.com/medic/cht-pipeline/issues/158)) ([ad88744](https://github.com/medic/cht-pipeline/commit/ad88744a4827824e290d474f4af8e82d64288e5f)), closes [#156](https://github.com/medic/cht-pipeline/issues/156)
+
+# [1.1.0](https://github.com/medic/cht-pipeline/compare/v1.0.0...v1.1.0) (2024-09-02)
+
+
+### Features
+
+* update postgres version to 16 ([9c44962](https://github.com/medic/cht-pipeline/commit/9c44962cd6fcb130638c1de11535e34506ca4c3e))
+
+# 1.0.0 (2024-08-27)
+
+
+### Bug Fixes
+
+* **#145:** docker compose instead of docker-compose ([b4ecea6](https://github.com/medic/cht-pipeline/commit/b4ecea6ac3abddd2d49e66dbd30a87409eae91dd)), closes [#145](https://github.com/medic/cht-pipeline/issues/145)
+
+
+### Features
+
+* **#148:** add automatic releases and versioning ([#152](https://github.com/medic/cht-pipeline/issues/152)) ([52cf12a](https://github.com/medic/cht-pipeline/commit/52cf12a50083a1f343a138cb4c5e1e5166644d7a)), closes [#148](https://github.com/medic/cht-pipeline/issues/148)
+* **#72:** Replace hardcoded values with env variables ([#73](https://github.com/medic/cht-pipeline/issues/73)) ([3dfa8e7](https://github.com/medic/cht-pipeline/commit/3dfa8e710ae45531f999788f17124a78d69d46d0)), closes [#72](https://github.com/medic/cht-pipeline/issues/72)

README.md

@@ -1,9 +1,7 @@
-# CHT Pipeline
-
-CHT Pipeline is a set of SQL queries that transform raw CouchDB data into a more useful format. It uses `dbt` to define the models that are translated into PostgreSQL tables or views, which makes it easier to query the data in the analytics platform of choice.
+A set of SQL queries that transform raw CouchDB data into a more useful format. It uses `dbt` to define the models that are translated into PostgreSQL tables or views, which makes it easier to query the data in the analytics platform of choice.
 
 ## Local Setup
-Follow the instructions in [the Local CHT Sync Setup documentation](https://docs.communityhealthtoolkit.org/apps/guides/data/analytics/setup/) to set up CHT Sync with CHT Pipeline locally.
+Follow the instructions in [the Local CHT Sync Setup documentation](https://docs.communityhealthtoolkit.org/apps/guides/data/analytics/setup/) to set up CHT Sync locally.
 
 ## Run dbt models unit tests locally
 
@@ -19,3 +17,19 @@ Follow the instructions in [the Local CHT Sync Setup documentation](https://docs
 # set environment variables, install dbt dependencies, seed data, run dbt, run test
 ./run_dbt_tests.sh
 ```
+
+## Release Process
+This repo has an automated release process where each feature/bug fix will be released immediately after it is merged to `main`. The release type is determined by the commit message format. Have a look at the development workflow in the [Contributor Handbook](https://docs.communityhealthtoolkit.org/contribute/code/workflow/) for more information.
+
+### Commit message format
+
+The commit format should follow the convention outlined in the [CHT docs](https://docs.communityhealthtoolkit.org/contribute/code/workflow/#commit-message-format).
+Examples are provided below.
+
+| Type        | Example commit message                                                                              | Release type |
+|-------------|-----------------------------------------------------------------------------------------------------|--------------|
+| Bug fixes   | fix(#123): rename column names                                                                      | patch        |
+| Performance | perf(#789): add new indexes                                                                         | patch        |
+| Features    | feat(#456): add new model                                                                           | minor        |
+| Non-code    | chore(#123): update README                                                                          | none         |
+| Breaking    | perf(#2): remove data_record model <br/> BREAKING CHANGE: form models should now read from new_model| major        |

commitlint.config.js

@@ -0,0 +1,2 @@
+module.exports = { extends: ['@commitlint/config-conventional'] };
+

dbt_project.yml

@@ -1,6 +1,9 @@
-name: 'pipeline'
+name: 'cht_pipeline_base'
 version: '0.0.1'
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
 profile: 'default'
+
+on-run-end:
+  - "{{ log_dbt_results(results) }}"

macros/log_dbt_results.sql

@@ -0,0 +1,44 @@
+{% macro log_dbt_results(results) %}
+  -- depends_on: {{ ref('dbt_results') }}
+  {%- if execute -%}
+    {%- set parsed_results = parse_dbt_results(results) -%}
+    {%- if parsed_results | length  > 0 -%}
+      {% set delete_dbt_results_query -%}
+        delete from {{ ref('dbt_results') }}
+      {%- endset -%}
+      {% set insert_dbt_results_query -%}
+        insert into {{ ref('dbt_results') }}
+          (
+            result_id,
+            invocation_id,
+            unique_id,
+            database_name,
+            schema_name,
+            name,
+            resource_type,
+            status,
+            execution_time,
+            rows_affected
+          ) values
+          {%- for parsed_result_dict in parsed_results -%}
+            (
+              '{{ parsed_result_dict.get('result_id') }}',
+              '{{ parsed_result_dict.get('invocation_id') }}',
+              '{{ parsed_result_dict.get('unique_id') }}',
+              '{{ parsed_result_dict.get('database_name') }}',
+              '{{ parsed_result_dict.get('schema_name') }}',
+              '{{ parsed_result_dict.get('name') }}',
+              '{{ parsed_result_dict.get('resource_type') }}',
+              '{{ parsed_result_dict.get('status') }}',
+              {{ parsed_result_dict.get('execution_time') }},
+              {{ parsed_result_dict.get('rows_affected') }}
+            ) {{- "," if not loop.last else "" -}}
+          {%- endfor -%}
+      {%- endset -%}
+      {%- do run_query(delete_dbt_results_query) -%}
+      {%- do run_query(insert_dbt_results_query) -%}
+    {%- endif -%}
+  {%- endif -%}
+  -- This macro is called from an on-run-end hook and therefore must return a query txt to run. Returning an empty string will do the trick
+  {{ return ('') }}
+{% endmacro %}

macros/parse_dbt_results.sql

@@ -0,0 +1,29 @@
+{% macro parse_dbt_results(results) %}
+  -- Create a list of parsed results
+  {%- set parsed_results = [] %}
+  -- Flatten results and add to list
+  {% for run_result in results %}
+    -- Convert the run result object to a simple dictionary
+    {% set run_result_dict = run_result.to_dict() %}
+    -- Get the underlying dbt graph node that was executed
+    {% set node = run_result_dict.get('node') %}
+    {% set rows_affected = run_result_dict.get('adapter_response', {}).get('rows_affected', 0) %}
+    {%- if not rows_affected -%}
+      {% set rows_affected = 0 %}
+    {%- endif -%}
+    {% set parsed_result_dict = {
+        'result_id': invocation_id ~ '.' ~ node.get('unique_id'),
+        'invocation_id': invocation_id,
+        'unique_id': node.get('unique_id'),
+        'database_name': node.get('database'),
+        'schema_name': node.get('schema'),
+        'name': node.get('name'),
+        'resource_type': node.get('resource_type'),
+        'status': run_result_dict.get('status'),
+        'execution_time': run_result_dict.get('execution_time'),
+        'rows_affected': rows_affected
+        }%}
+    {% do parsed_results.append(parsed_result_dict) %}
+  {% endfor %}
+  {{ return(parsed_results) }}
+{% endmacro %}

models/contacts/contact.sql

@@ -14,8 +14,8 @@
 }}
 
 SELECT
-  _id as uuid,
-  saved_timestamp,
+  document_metadata.uuid as uuid,
+  document_metadata.saved_timestamp,
   to_timestamp((NULLIF(doc ->> 'reported_date'::text, ''::text)::bigint / 1000)::double precision) AS reported,
   doc->'parent'->>'_id' AS parent_uuid,
   doc->>'name' AS name,
@@ -24,9 +24,15 @@ SELECT
   doc->>'alternative_phone' AS phone2,
   doc->>'is_active' AS active,
   doc->>'notes' AS notes,
-  doc->>'contact_id' AS contact_id
-FROM {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}
-WHERE doc->>'type' IN ('contact', 'clinic', 'district_hospital', 'health_center', 'person')
+  doc->>'contact_id' AS contact_id,
+  NULLIF(doc->> 'muted', '') AS muted
+FROM {{ ref('document_metadata') }} document_metadata
+INNER JOIN
+  {{ source('couchdb', env_var('POSTGRES_TABLE')) }} source_table
+  ON source_table._id = document_metadata.uuid
+WHERE
+  document_metadata.doc_type IN ('contact', 'clinic', 'district_hospital', 'health_center', 'person')
+  AND document_metadata._deleted = false
 {% if is_incremental() %}
-  AND saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }}
+  AND document_metadata.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }}
 {% endif %}