Add dbt_dev environment

CONTRIBUTING.md

@@ -35,3 +35,18 @@ terraform-docs .
 ```
 
 This will replace the readme file at `deploy/infrastructure/README.md` and `deploy/meltano/README.md` with any changes made to the module and header docs.
+
+## Developing transforms
+
+To develop transforms, you'll need to duplicate `.env.template` as `.env` and ensure that at least these env vars are declared:
+
+- `SNOWFLAKE_USER`
+- `SNOWFLAKE_PASSWORD`
+
+## Incremental build with `--defer` option
+
+```console
+meltano invoke dbt-snowflake:seed
+meltano invoke dbt-snowflake:snapshot
+meltano invoke dbt-snowflake:build
+```

data/.env.template

@@ -19,10 +19,9 @@ PERMISSION_BOT_PASSWORD="****"
 PERMISSION_BOT_ACCOUNT="****"
 
 # Snowflake MELTANO User Password
+SNOWFLAKE_USER="...."
 SNOWFLAKE_PASSWORD="****"
-
-# dbt Snowflake Password
-DBT_SNOWFLAKE_PASSWORD="****"
+SNOWFLAKE_ROLE="DEVELOPER"
 
 # Slack Webhooks
 TARGET_APPRISE_SINGER_ACTIVITY_URIS=["https://hooks.slack.com/services/{}/{}/{}"]

data/.sqlfluff

@@ -17,7 +17,9 @@ profile = meltano
 tab_space_size = 4
 max_line_length = 80
 indent_unit = space
-comma_style = trailing
+
+[sqlfluff:layout:type:comma]
+line_position = trailing
 
 [sqlfluff:rules:L010] # Keywords
 capitalisation_policy = upper

data/environments/dbt_dev.meltano.yml

@@ -0,0 +1,42 @@
+environments:
+- name: dbt_dev
+  annotations:
+    docs:
+      description: >
+        Environment for developing dbt transforms separately from EL.
+  config:
+    plugins:
+      utilities:
+      - name: dbt-snowflake
+        config:
+          user: ${SNOWFLAKE_USER}
+          role: ${SNOWFLAKE_USER}
+          warehouse: CORE
+          skip_pre_invoke: true
+          database: USERDEV_PROD
+          database_prep: USERDEV_PREP
+          target_schema_prefix: ${SNOWFLAKE_USER}_
+      - name: sqlfluff
+        config:
+          user: ${SNOWFLAKE_USER}
+      - name: great_expectations
+        config:
+          prod_database: USERDEV_PROD
+          raw_database: USERDEV_RAW
+          username: ${SNOWFLAKE_USER}
+          role: ${SNOWFLAKE_USER}
+          warehouse: CORE
+  env:
+    USER_PREFIX: ${SNOWFLAKE_USER}
+    SUPERSET_API_URL: http://localhost:8088
+    SUPERSET_USER: admin
+    SUPERSET_PASS: admin
+    # https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html
+    AIRFLOW__CORE__PLUGINS_FOLDER: $MELTANO_PROJECT_ROOT/orchestrate/plugins_local
+    AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: '30'
+    AIRFLOW_VAR_MELTANO_ENVIRONMENT: userdev
+    AIRFLOW_VAR_OPERATOR_TYPE: bash
+    # Secrets via KMS
+    KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem
+    KMS_DOTENV_PATH: .env
+    KMS_SECRETS_PATH: secrets.yml

data/environments/userdev.meltano.yml

@@ -22,8 +22,8 @@ environments:
       - name: tap-snowflake
         config:
           dbname: USERDEV_PROD
-          user: ${USER_PREFIX}
-          role: ${USER_PREFIX}
+          user: ${SNOWFLAKE_USER}
+          role: ${SNOWFLAKE_USER}
           warehouse: CORE
       - name: tap-snowflake-metrics-legacy
         config:
@@ -85,7 +85,7 @@ environments:
           role: ${USER_PREFIX}
           warehouse: CORE
   env:
-    USER_PREFIX: PNADOLNY
+    USER_PREFIX: ${SNOWFLAKE_USER}
     SUPERSET_API_URL: http://localhost:8088
     SUPERSET_USER: admin
     SUPERSET_PASS: admin
@@ -97,4 +97,4 @@ environments:
     # Secrets via KMS
     KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem
     KMS_DOTENV_PATH: .env
-    KMS_SECTRETS_PATH: secrets.yml
+    KMS_SECRETS_PATH: secrets.yml

data/meltano.yml

@@ -1,5 +1,5 @@
 version: 1
-default_environment: userdev
+default_environment: dbt_dev
 send_anonymous_usage_stats: false
 project_id: c15e971a-d318-4a9d-979b-1039ce5fd1b1
 include_paths:
@@ -9,3 +9,5 @@ include_paths:
 - ./orchestrate/*.meltano.yml
 - ./transform/*.meltano.yml
 - ./utilities/*.meltano.yml
+ff:
+  strict_env_var_mode: true

data/transform/profiles/snowflake/profiles.yml

@@ -18,6 +18,16 @@ meltano:
       warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}"
       database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}"
       schema: "DEFAULT"
+    dbt_dev:
+      type: snowflake
+      threads: "{{ env_var('DBT_THREADS', 6) | as_number }}"
+      account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_USER') }}"
+      password: "{{ env_var('SNOWFLAKE_PASSWORD') }}"
+      role: "{{ env_var('DBT_SNOWFLAKE_ROLE', env_var('SNOWFLAKE_USER', )) }}"
+      warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}"
+      database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}"
+      schema: "DEFAULT"
     userdev:
       type: snowflake
       threads: 6

data/transform/transformers.meltano.yml

@@ -38,6 +38,7 @@ plugins:
       run_slack_notifications: run --select +publish.slack_notifications.*
       run_hubspot_publish: run --select publish.hubspot.*
       test_hubspot_publish: test --select publish.hubspot.*
+      build: run --fail-fast --defer --state=${MELTANO_PROJECT_ROOT}/.meltano/transformers/dbt/target/
 
     config:
       account: epa06486

data/utilities/utilities.meltano.yml

@@ -68,9 +68,9 @@ plugins:
     executable: kms
     commands:
       encrypt: encrypt $KMS_PUBLIC_KEY_PATH --dotenv-path $KMS_DOTENV_PATH --output-path
-        $KMS_SECTRETS_PATH
+        $KMS_SECRETS_PATH
       decrypt: decrypt $KMS_KEY_ID --input-path $KMS_SECRETS_PATH --output-path $KMS_DOTENV_PATH
     env:
       KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem
       KMS_DOTENV_PATH: .env
-      KMS_SECTRETS_PATH: secrets.yml
+      KMS_SECRETS_PATH: secrets.yml