21 add dbt unit test (#22)

* Add test setup

* Use env variables in profile and add script to run tests

* Add simple test on root model and add tear down script

* Add github test workflow

* Add some single tests, dependencies and, seeds

* Rename models yml files to schema

* Add properties to seeds, macro select_table and use macro on couchdb model

* Rename schemas properties files, add more tests, rearange dependencies

* Add sudo command

* Move package.yml to base directory

* Add instructions on how to run unit tests locally

* Change titles structure

---------

Co-authored-by: Maria Lorena Rodriguez Viruel <[email protected]>

.github/workflows/test.yml

@@ -0,0 +1,43 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  dbt-unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16.x
+      - name: Get Docker Hub username
+        id: get-docker-hub-username
+        run: echo '::set-output name=dockerhub_username::${{ secrets.DOCKERHUB_USERNAME }}'
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        if: steps.get-docker-hub-username.outputs.dockerhub_username
+      - name: Install PostgreSQL client
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes postgresql-client
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install dependencies
+        run: |
+          pip install dbt-postgres
+          dbt deps
+      - name: Setup test
+        run: ./setup.sh
+        working-directory: ./tests
+      - name: Run dbt and tests
+        run: ./run_dbt_tests.sh
+        working-directory: ./tests
+      - name: Cleanup
+        run: ./tear_down.sh
+        working-directory: ./tests

.gitignore

@@ -1,4 +1,7 @@
 
 target/
 dbt_modules/
+dbt_packages/
 logs/
+*/.env
+*/dbt_packages/

README.md

@@ -6,4 +6,33 @@ CHT Pipeline is a tool used to define data models for transforming the raw data
 # Dev installation instructions
 
 1. Follow the instructions in [cht-sync](https://github.com/medic/cht-sync) to set up the required environment variables
-1. Run `docker-compose up` in the cht-sync directory to run the pipeline. Please ensure the `DATAEMON_INITAL_PACKAGE` env variable is set to your preferred branch of the cht-pipeline repo. 
+1. Run `docker-compose up` in the cht-sync directory to run the pipeline. Please ensure the `DATAEMON_INITAL_PACKAGE` env variable is set to your preferred branch of the cht-pipeline repo.
+
+# Test
+## Run dbt models unit tests locally
+### Prerequisites
+
+- `Docker`
+- `PostgreSQL client`
+- `dbt-postgres`
+- `Python`
+
+1. Navigate to tests folder
+2. Run setup script:
+
+```sh
+# Set environment variables, create postgres database, schema and user:
+./setup.sh
+```
+
+3. Run dbt test
+
+```sh
+# set environment variables, install dbt dependencies, seed data, run dbt, run test
+./run_dbt_tests.sh
+```
+
+4. Clean up:
+```sh
+./tear_down.sh
+```

dbt_project.yml

@@ -1,29 +1,27 @@
-
 # Name your project! Project names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'pipeline'
-version: '0.0.1'
+name: "pipeline"
+version: "0.0.1"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
-profile: 'default'
+profile: "default"
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
-source-paths: ["models"]
+model-paths: ["models"]
 analysis-paths: ["analysis"]
 test-paths: ["tests"]
-data-paths: ["data"]
+seed-paths: ["seeds"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
-target-path: "target"  # directory which will store compiled SQL files
-clean-targets:         # directories to be removed by `dbt clean`
-    - "target"
-    - "dbt_modules"
-
+target-path: "target" # directory which will store compiled SQL files
+clean-targets: # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_modules"
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
@@ -32,11 +30,10 @@ clean-targets:         # directories to be removed by `dbt clean`
 # as tables. These settings can be overridden in the individual model files
 # using the `{{ config(...) }}` macro.
 models:
-    type_partitions:
-        root:
-            materialized: view
-        types:
-            materialized: incremental
-        contact_views:
-            materialized: view
-
+  type_partitions:
+    root:
+      materialized: view
+    types:
+      materialized: incremental
+    contact_views:
+      materialized: view

macros/select_table.sql

@@ -0,0 +1,13 @@
+{% macro select_table(source_table, test_table) %}
+
+  {% if target.name == 'test' %}
+
+    {{ return(test_table) }}
+
+  {% else %}
+
+    {{ return(source_table) }}
+
+  {% endif %}
+
+{% endmacro %}

models/contact_views/contact_views.yml

@@ -0,0 +1,25 @@
+version: 2
+
+models:
+  - name: raw_contacts
+    columns:
+      - name: doc
+        tests:
+          - not_null
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('couchdb')
+  - name: contactview_metadata
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+      - name: type
+        tests:
+          - not_null
+          - accepted_values:
+              values:
+                [contact, clinic, district_hospital, health_center, person]
+    tests:
+      - dbt_utils.equal_rowcount:
+          compare_model: ref('raw_contacts')

models/households/households.yml

@@ -0,0 +1,7 @@
+version: 2
+
+models:
+  - name: household_visits
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('couchdb')

models/reports_tables/reports_tables.yml

@@ -0,0 +1,25 @@
+version: 2
+
+models:
+  - name: reports_by_location
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+      - name: longitude
+        tests:
+          - not_null
+      - name: latitude
+        tests:
+          - not_null
+    tests:
+      - dbt_utils.equal_rowcount:
+          compare_model: ref('reports')
+  - name: reports
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('couchdb')

models/root/couchdb.sql

@@ -1,3 +1,5 @@
+{% set import_couchdb_data = select_table(source('couchdb','couchdb'), ref('couchdb_test_data')) %}
+
 {{
     config(
         materialized = 'view',
@@ -10,4 +12,4 @@
 SELECT
     doc->>'type' AS type,
     *
-FROM {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}
+FROM {{ import_couchdb_data }}

models/root/root.yml

@@ -0,0 +1,22 @@
+version: 2
+
+sources:
+  - name: couchdb
+    database: data
+    schema: v1
+    tables:
+      - name: couchdb
+models:
+  - name: couchdb
+    columns:
+      - name: _id
+        tests:
+          - not_null
+      - name: _rev
+        tests:
+          - not_null
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - _id
+            - _rev

models/types/types.yml

@@ -0,0 +1,11 @@
+version: 2
+
+models:
+  - name: data_record
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('couchdb')
+  - name: person
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('couchdb')

models/user_tables/user_tables.yml

@@ -0,0 +1,9 @@
+version: 2
+
+models:
+  - name: chws
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('couchdb')
+      - dbt_utils.equal_rowcount:
+          compare_model: ref('person')

packages.yml

@@ -0,0 +1,3 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: 1.1.1