Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use views for staging and refresh only in CI deployments #525

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/prod_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,4 @@ jobs:
# Seed Production
- run: meltano install utility dbt-snowflake
- run: meltano run dbt-snowflake:run_dbt_artifacts dbt-snowflake:seed
- run: meltano run dbt-snowflake:materialize_views
1 change: 1 addition & 0 deletions data/orchestrate/orchestrators.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ jobs:
- name: slack_notifications
tasks:
- dbt-snowflake:run_slack_notifications
- dbt-snowflake:test_slack_notifications
- tap-snowflake-singer-activity target-apprise

- name: dbt_docs
Expand Down
14 changes: 14 additions & 0 deletions data/transform/models/publish/slack_notifications/schema.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
version: 2

models:
- name: slack_alerts
description: This table includes the message to be sent to Slack for the singer-ecosystem-activity channel.
columns:
- name: title
description: The Slack message title.
tests:
- not_null
- name: body
description: The body of the Slack message, in markdown format.
tests:
- not_null
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
{{ config(materialized='table') }}

WITH source AS (

SELECT
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
{{ config(materialized='table') }}

WITH source AS (

SELECT
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
{{ config(materialized='table') }}

WITH source AS (

SELECT
Expand Down
2 changes: 0 additions & 2 deletions data/transform/models/staging/gitlab/stg_gitlab__issues.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
{{ config(materialized='table') }}

WITH source AS (

SELECT
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
{{ config(materialized='table') }}

WITH source AS (

SELECT
Expand Down
2 changes: 0 additions & 2 deletions data/transform/models/staging/gitlab/stg_gitlab__projects.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
{{ config(materialized='table') }}

WITH source AS (

SELECT
Expand Down
23 changes: 13 additions & 10 deletions data/transform/transformers.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,26 @@ plugins:
\ False, 'schema_list': []}\""
description: Clones all production schemas in the PREP database that the user
has access to.

run_dbt_artifacts: run --select dbt_artifacts.*

run_marts: run --select marts.* common.* --exclude staging.*,dbt_artifacts.*
test_marts: test --select marts.* common.* --exclude staging.*,dbt_artifacts.*
run_marts: run --select marts.* common.* --exclude config.materialized:view
test_marts: test --select marts.* common.*

run_staging_meltanohub: run --select staging.meltanohub.*
run_snapshot_meltanohub: snapshot --select meltanohub.*
test_source_meltanohub: test --select source:tap_meltanohub
test_staging_meltanohub: test --select staging.meltanohub.*,test_type:generic

run_snapshot_spreadsheets_anywhere: snapshot --select spreadsheets_anywhere.*
test_source_spreadsheets_anywhere: test --select source:tap_spreadsheets_anywhere

run_staging_snowplow: run --select staging.snowplow.*
test_staging_snowplow: test --select staging.snowplow.*,test_type:generic

run_staging_github_search: run --select staging.github_search.*
test_staging_github_search: test --select staging.github_search.*,test_type:generic

run_staging_github_meltano: run --select staging.github_meltano.*
test_staging_github_meltano: test --select staging.github_meltano.*,test_type:generic

Expand All @@ -41,10 +41,13 @@ plugins:
run_staging_google_analytics: run --select staging.google_analytics.*
test_staging_google_analytics: test --select staging.google_analytics.*,test_type:generic

run_hub_metrics: run --select publish.meltano_hub.*
run_hub_metrics: run --select publish.meltano_hub.* --exclude config.materialized:view
test_hub_metrics: test --select publish.meltano_hub.*

run_slack_notifications: run --select publish.slack_notifications.*
run_slack_notifications: run --select publish.slack_notifications.* --exclude config.materialized:view
test_slack_notifications: test --select publish.slack_notifications.*

materialize_views: run --select config.materialized:view
config:
account: epa06486
database_raw: RAW
Expand Down