From 37b0472d4bac2a87271146310d2f69e8a2faea3a Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 13:34:29 -0600 Subject: [PATCH 01/23] Add macros for postgres --- macros/insert_into_metadata_table.sql | 11 ++++ macros/upload_exposures.sql | 49 +++++++++++++++ macros/upload_invocations.sql | 87 +++++++++++++++++++++++++++ macros/upload_model_executions.sql | 77 ++++++++++++++++++++++++ macros/upload_models.sql | 50 +++++++++++++++ macros/upload_seed_executions.sql | 77 ++++++++++++++++++++++++ macros/upload_seeds.sql | 43 +++++++++++++ macros/upload_snapshot_executions.sql | 77 ++++++++++++++++++++++++ macros/upload_snapshots.sql | 47 +++++++++++++++ macros/upload_sources.sql | 43 +++++++++++++ macros/upload_test_executions.sql | 72 ++++++++++++++++++++++ macros/upload_tests.sql | 37 ++++++++++++ 12 files changed, 670 insertions(+) diff --git a/macros/insert_into_metadata_table.sql b/macros/insert_into_metadata_table.sql index 235d2326..adc46b4a 100644 --- a/macros/insert_into_metadata_table.sql +++ b/macros/insert_into_metadata_table.sql @@ -34,5 +34,16 @@ {%- endmacro %} +{% macro postgres__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} + + {% set insert_into_table_query %} + insert into {{database_name}}.{{ schema_name }}.{{ table_name }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + {% macro default__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} {%- endmacro %} diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index daa86fea..d48a42b6 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -80,3 +80,52 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_exposures_dml_sql(exposures) -%} + {% if exposures != [] %} + + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'name', + 'type', + 'owner', + 'maturity', + 'path', + 'description', + 'url', + 'package_name', + 'depends_on_nodes', + 'tags', + 'all_results', + ] %} + + {% set exposure_values %} + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + $${{ exposure.unique_id }}$$, {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + $${{ exposure.name }}$$, {# name #} + '{{ exposure.type }}', {# type #} + $${{ tojson(exposure.owner) }}$$, {# owner #} + '{{ exposure.maturity }}', {# maturity #} + $${{ exposure.original_file_path }}$$, {# path #} + $${{ exposure.description }}$$, {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + $${{ tojson(exposure.depends_on.nodes) }}$$, {# depends_on_nodes #} + $${{ tojson(exposure.tags) }}$$, {# tags #} + $${{ tojson(exposure) }}$$ {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index b1d6e7bc..b12cc1f3 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -159,3 +159,90 @@ {{ invocation_values }} {% endmacro -%} + +{% macro postgres__get_invocations_dml_sql() -%} + {% set columns = [ + 'command_invocation_id', + 'dbt_version', + 'project_name', + 'run_started_at', + 'dbt_command', + 'full_refresh_flag', + 'target_profile_name', + 'target_name', + 'target_schema', + 'target_threads', + 'dbt_cloud_project_id', + 'dbt_cloud_job_id', + 'dbt_cloud_run_id', + 'dbt_cloud_run_reason_category', + 'dbt_cloud_run_reason', + 'env_vars', + 'dbt_vars', + 'invocation_args', + 'dbt_custom_envs', + ] %} + + {% set invocation_values %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + {{ flags.FULL_REFRESH }}, {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #} + '{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #} + '{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #} + '{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #} + '{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","\\'") }}', {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} + {% endfor %} + $${{ tojson(env_vars_dict) }}$$, {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} + {% endfor %} + $${{ tojson(dbt_vars_dict) }}$$, {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + + {% if invocation_args_dict.vars %} + {# vars - different format for pre v1.5 (yaml vs list) #} + {% if invocation_args_dict.vars is string %} + {# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #} + {% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %} + {% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %} + {% endif %} + {% endif %} + + $${{ tojson(invocation_args_dict) }}$$, {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: value}) %} + {% endfor %} + $${{ tojson(metadata_env) }}$$ {# dbt_custom_envs #} + + ) + {% endset %} + {{ "(" ~ columns | join(', ') + ")" }} + VALUES + {{ invocation_values }} + +{% endmacro -%} diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index 14917d1a..d63f8d0e 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -216,3 +216,80 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'was_full_refresh', + 'thread_id', + 'status', + 'compile_started_at', + 'query_completed_at', + 'total_node_runtime', + 'rows_affected', + 'materialization', + 'schema', + 'name', + 'alias', + 'message', + 'adapter_response', + ]%} + {% set model_execution_values %} + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ model_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_models.sql b/macros/upload_models.sql index b8fce69e..0c9b4480 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -81,3 +81,53 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_models_dml_sql(models) -%} + {% if models != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'database', + 'schema', + 'name', + 'depends_on_nodes', + 'package_name', + 'path', + 'checksum', + 'materialization', + 'tags', + 'meta', + 'alias', + 'all_results', + ] %} + {% set model_values %} + {% for model in models -%} + {% do model.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + '{{ tojson(model.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + $${{ model.original_file_path | replace('\\', '\\\\') }}$$, {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + '{{ tojson(model.tags) }}', {# tags #} + $${{ model.config.meta }}$$, {# meta #} + '{{ model.alias }}', {# alias #} + $${{ tojson(model) }}$$ {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index a7ada664..81d9f4e5 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -214,3 +214,80 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'was_full_refresh', + 'thread_id', + 'status', + 'compile_started_at', + 'query_completed_at', + 'total_node_runtime', + 'rows_affected', + 'materialization', + 'schema', + 'name', + 'alias', + 'message', + 'adapter_response', + ]%} + {% set seed_execution_values %} + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Databricks #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} + parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ seed_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index 61683f75..7b0b6d30 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -74,3 +74,46 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_seeds_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'database', + 'schema', + 'name', + 'package_name', + 'path', + 'checksum', + 'meta', + 'alias', + 'all_results', + ] %} + {% set seed_values %} + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + $${{ tojson(seed.config.meta) }}$$, {# meta #} + '{{ seed.alias }}', {# alias #} + $${{ tojson(seed) }}$$ {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index 4903f48e..2d43ff50 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -214,3 +214,80 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'was_full_refresh', + 'thread_id', + 'status', + 'compile_started_at', + 'query_completed_at', + 'total_node_runtime', + 'rows_affected', + 'materialization', + 'schema', + 'name', + 'alias', + 'message', + 'adapter_response', + ]%} + {% set snapshot_execution_values %} + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, {# rows_affected #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ snapshot_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index cac08abc..6e9a7029 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -81,3 +81,50 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_snapshots_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'database', + 'schema', + 'name', + 'depends_on_nodes', + 'package_name', + 'path', + 'checksum', + 'strategy', + 'meta', + 'alias', + 'all_results', + ]%} + {% set snapshot_values %} + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + $${{ tojson(snapshot.depends_on.nodes) }}$$, {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + $${{ tojson(snapshot.config.meta) }}$$, {# meta #} + '{{ snapshot.alias }}', {# alias #} + $${{ tojson(snapshot) }}$$ {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index a624ea4b..0487425c 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -70,3 +70,46 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_sources_dml_sql(sources) -%} + {% if sources != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'database', + 'schema', + 'source_name', + 'loader', + 'name', + 'identifier', + 'loaded_at_field', + 'freshness', + 'all_results', + ] %} + {% set source_values %} + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + $${{ source.loaded_at_field }}$$, {# loaded_at_field #} + $${{ tojson(source.freshness) }}$$, {# freshness #} + $${{ tojson(source) }}$$ {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index 1f0f43d7..ae5cf323 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -131,3 +131,75 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'was_full_refresh', + 'thread_id', + 'status', + 'compile_started_at', + 'query_completed_at', + 'total_node_runtime', + 'rows_affected', + 'failures', + 'message', + 'adapter_response', + ]%} + {% set test_execution_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% if test.timing != [] %} + {% for stage in test.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in test.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ test.execution_time }}, {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} + parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index 5d7c268d..493b65bd 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -61,3 +61,40 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_tests_dml_sql(tests) -%} + {% if tests != [] %} + {% set columns = [ + 'command_invocation_id', + 'node_id', + 'run_started_at', + 'name', + 'depends_on_nodes', + 'package_name', + 'test_path', + 'tags', + 'all_results', + ] %} + {% set test_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + $${{ tojson(test.depends_on.nodes) }}$$, {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} + $${{ tojson(test.tags) }}$$, {# tags #} + $${{ tojson(test) }}$$ {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ "(" ~ columns | join(', ') ~ ")"}} + VALUES + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} From 37f48baf92c019c5e1f215c71c403bdc5e1c8205 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 15:49:48 -0600 Subject: [PATCH 02/23] Add steps for postgres --- tox.ini | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tox.ini b/tox.ini index f311d76d..fb08390e 100644 --- a/tox.ini +++ b/tox.ini @@ -226,3 +226,26 @@ commands = dbt deps dbt build --exclude snapshot --target spark +[testenv:integration_postgres_1_3_0] +changedir = integration_test_project +deps = dbt-core~=1.3.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_snowflake_1_4_0] +changedir = integration_test_project +deps = dbt-core~=1.4.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_snowflake_1_5_0] +changedir = integration_test_project +deps = dbt-core~=1.5.0 +commands = + dbt clean + dbt deps + dbt build --target postgres From 1870a84902587b4eba1ea6bf93556c7f12432b88 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 15:50:01 -0600 Subject: [PATCH 03/23] Add postgres entry --- integration_test_project/profiles.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index e8548e3f..1c79c489 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -43,3 +43,13 @@ dbt_artifacts: timeout_seconds: 300 priority: interactive retries: 1 + postgres: + type: postgres + host: postgres + user: postgres + password: postgres + port: 5432 + dbname: postgres + schema: public + threads: 8 + From c91cecc6a4fbeb232cbdf4be5eb55ad4b6dd2eda Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 15:50:17 -0600 Subject: [PATCH 04/23] Add a postgres service to action --- .github/workflows/main_test_package.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 2202c05c..2251b284 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,13 +34,24 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] version: ["1_3_0", "1_4_0", "1_5_0"] runs-on: ubuntu-latest permissions: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - name: Checkout uses: actions/checkout@v3 From a9931c3eb53789166c7d7fc16a9fe20e9f8dca0b Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 15:50:59 -0600 Subject: [PATCH 05/23] Just test postgres for now --- .github/workflows/main_test_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 2251b284..9f55d223 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["postgres"] version: ["1_3_0", "1_4_0", "1_5_0"] runs-on: ubuntu-latest permissions: From 0acb98e1e66be8032bff6b213fc12e1ed623ab40 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 15:52:51 -0600 Subject: [PATCH 06/23] Fix duplicate section --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fb08390e..11db7a07 100644 --- a/tox.ini +++ b/tox.ini @@ -234,7 +234,7 @@ commands = dbt deps dbt build --target postgres -[testenv:integration_snowflake_1_4_0] +[testenv:integration_postgres_1_4_0] changedir = integration_test_project deps = dbt-core~=1.4.0 commands = @@ -242,7 +242,7 @@ commands = dbt deps dbt build --target postgres -[testenv:integration_snowflake_1_5_0] +[testenv:integration_postgres_1_5_0] changedir = integration_test_project deps = dbt-core~=1.5.0 commands = From 927e50b2da2af4fd98e0c726df01c4661cfa2420 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 15:55:05 -0600 Subject: [PATCH 07/23] Install correct package --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 11db7a07..67d0061b 100644 --- a/tox.ini +++ b/tox.ini @@ -228,7 +228,7 @@ commands = [testenv:integration_postgres_1_3_0] changedir = integration_test_project -deps = dbt-core~=1.3.0 +deps = dbt-postgres~=1.3.0 commands = dbt clean dbt deps @@ -236,7 +236,7 @@ commands = [testenv:integration_postgres_1_4_0] changedir = integration_test_project -deps = dbt-core~=1.4.0 +deps = dbt-postgres~=1.4.0 commands = dbt clean dbt deps @@ -244,7 +244,7 @@ commands = [testenv:integration_postgres_1_5_0] changedir = integration_test_project -deps = dbt-core~=1.5.0 +deps = dbt-postgres~=1.5.0 commands = dbt clean dbt deps From 7e0173a09ba497a4a954fbe1ab08d4746d54e9c5 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 15:56:33 -0600 Subject: [PATCH 08/23] Update host to localhost --- integration_test_project/profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index 1c79c489..c33d5b3f 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -45,7 +45,7 @@ dbt_artifacts: retries: 1 postgres: type: postgres - host: postgres + host: localhost user: postgres password: postgres port: 5432 From bc91434409a1cc7c289118348f599412616a24ed Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 16:00:06 -0600 Subject: [PATCH 09/23] Update profiles.yml to env vars --- integration_test_project/profiles.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index c33d5b3f..9ac7adbd 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -45,10 +45,10 @@ dbt_artifacts: retries: 1 postgres: type: postgres - host: localhost + host: "{{ env_var('POSTGRES_HOST') }}" user: postgres password: postgres - port: 5432 + port: "{{ env_var('POSTGRES_PORT') }}" dbname: postgres schema: public threads: 8 From e906e8a118ff966fd240b925f97b9160b07ccf1c Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 16:00:21 -0600 Subject: [PATCH 10/23] Add postgres env vars to step --- .github/workflows/main_test_package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 9f55d223..460e5c5f 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -74,6 +74,8 @@ jobs: - name: Run ${{ matrix.warehouse }} Tests env: DBT_VERSION: ${{ matrix.version }} + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 run: tox -e integration_${{ matrix.warehouse }}_${{ matrix.version }} # Databricks doesn't like the matrix strategy, so moving back to the old integration testing without versioning From f5d9677544fd28b6f1ae08a792e280f591adf21a Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 16:03:08 -0600 Subject: [PATCH 11/23] Try again --- .github/workflows/main_test_package.yml | 4 +++- integration_test_project/profiles.yml | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 460e5c5f..510bd902 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -51,6 +51,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + ports: + - 5432:5432 steps: - name: Checkout @@ -74,7 +76,7 @@ jobs: - name: Run ${{ matrix.warehouse }} Tests env: DBT_VERSION: ${{ matrix.version }} - POSTGRES_HOST: postgres + POSTGRES_HOST: localhost POSTGRES_PORT: 5432 run: tox -e integration_${{ matrix.warehouse }}_${{ matrix.version }} diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index 9ac7adbd..c33d5b3f 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -45,10 +45,10 @@ dbt_artifacts: retries: 1 postgres: type: postgres - host: "{{ env_var('POSTGRES_HOST') }}" + host: localhost user: postgres password: postgres - port: "{{ env_var('POSTGRES_PORT') }}" + port: 5432 dbname: postgres schema: public threads: 8 From cd1fb78e66edb447dc149662619aa4dad822b1ee Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 16:57:18 -0600 Subject: [PATCH 12/23] Remove parse_json func --- macros/upload_seed_executions.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index 81d9f4e5..951c161e 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -278,8 +278,8 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} From a74610fa6926efcf4d44ea40b457f3c5eb37787b Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 17:00:28 -0600 Subject: [PATCH 13/23] Another fix --- macros/upload_model_executions.sql | 2 +- macros/upload_snapshot_executions.sql | 2 +- macros/upload_test_executions.sql | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index d63f8d0e..f8e591cb 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -280,7 +280,7 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} + $${{ model.message }}$$, {# message #} $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index 2d43ff50..a1004edd 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -278,7 +278,7 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} + $${{ model.message }}$$, {# message #} $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index ae5cf323..497c6601 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -189,8 +189,8 @@ {{ test.execution_time }}, {# total_node_runtime #} null, {# rows_affected not available in Databricks #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} - '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# adapter_response #} + $${{ test.message }}$$, {# message #} + $${{ tojson(test.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} From 70819c3e29593491ff2b2b65a02a4faf03b429fd Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 17:04:06 -0600 Subject: [PATCH 14/23] Another fix --- macros/upload_invocations.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index b12cc1f3..ad729efd 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -200,7 +200,7 @@ '{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #} '{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #} '{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #} - '{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","\\'") }}', {# dbt_cloud_run_reason #} + $${{ env_var('DBT_CLOUD_RUN_REASON', '') }}$$, {# dbt_cloud_run_reason #} {% if var('env_vars', none) %} {% set env_vars_dict = {} %} From b5b67e3b5d3a43c249f386ef4fa90fae40d04118 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 17:08:38 -0600 Subject: [PATCH 15/23] Alias the subquery --- integration_test_project/tests/singular_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql index 67a2508d..a1e49e8c 100644 --- a/integration_test_project/tests/singular_test.sql +++ b/integration_test_project/tests/singular_test.sql @@ -1 +1 @@ -select 1 as failures from (select 2) where 1 = 2 +select 1 as failures from (select 2) as foo where 1 = 2 From a5e28e27fbf834a9b737ce1395b701c8077dc7fc Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 17:11:20 -0600 Subject: [PATCH 16/23] Add back other CDW [skip-ci] --- .github/workflows/main_test_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 510bd902..481ed759 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["postgres"] + warehouse: ["snowflake", "bigquery", "postgres"] version: ["1_3_0", "1_4_0", "1_5_0"] runs-on: ubuntu-latest permissions: From 4c13b2a98b84d6313243dd9eac278a3250ff0a47 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 20:23:16 -0600 Subject: [PATCH 17/23] Add postgres as supported DW [skip ci] --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 94853de2..3c863d5d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The package currently supports - Spark :white_check_mark: - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: +- Postgres :white_check_mark: Models included: From ddc6aff1f204637c14551d3283e6c5f6412ef6ef Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 20:24:31 -0600 Subject: [PATCH 18/23] Test removing env vars --- .github/workflows/main_test_package.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 481ed759..26332df7 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -76,8 +76,6 @@ jobs: - name: Run ${{ matrix.warehouse }} Tests env: DBT_VERSION: ${{ matrix.version }} - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 run: tox -e integration_${{ matrix.warehouse }}_${{ matrix.version }} # Databricks doesn't like the matrix strategy, so moving back to the old integration testing without versioning From 6fc91e752e36b7843697187004af54c311e8a2e8 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 20:26:14 -0600 Subject: [PATCH 19/23] Another test --- .github/workflows/main_test_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 26332df7..82e4dc88 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["postgres"] version: ["1_3_0", "1_4_0", "1_5_0"] runs-on: ubuntu-latest permissions: From dc3ffae5328659e08a8d3436beb25339792ed337 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 12 Jul 2023 20:28:31 -0600 Subject: [PATCH 20/23] Add back DWs --- .github/workflows/main_test_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 82e4dc88..26332df7 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["postgres"] + warehouse: ["snowflake", "bigquery", "postgres"] version: ["1_3_0", "1_4_0", "1_5_0"] runs-on: ubuntu-latest permissions: From 778a435ddee146002d1c34840caab42e5154a0e3 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 29 Sep 2023 14:10:06 +0100 Subject: [PATCH 21/23] Add missing CI tests --- .github/workflows/ci_test_package.yml | 30 +++++++++++++++++-- .../insert_into_metadata_table.sql | 13 ++++++++ tox.ini | 17 +++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index edd8cdc9..b02997ee 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] runs-on: ubuntu-latest environment: name: Approve Integration Tests @@ -45,6 +45,19 @@ jobs: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Get latest release uses: rez0n/actions-github-release@main @@ -100,7 +113,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] runs-on: ubuntu-latest @@ -110,6 +123,19 @@ jobs: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - uses: actions/setup-python@v4 with: diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 2ec4d5d5..b1ad9993 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -45,5 +45,18 @@ {%- endmacro %} +{%- endmacro %} + +{% macro postgres__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} diff --git a/tox.ini b/tox.ini index f76fca6e..8add76ba 100644 --- a/tox.ini +++ b/tox.ini @@ -250,6 +250,14 @@ commands = dbt deps dbt build --exclude snapshot --target spark +[testenv:integration_postgres] +changedir = integration_test_project +deps = dbt-postgres~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + [testenv:integration_postgres_1_3_0] changedir = integration_test_project deps = dbt-postgres~=1.3.0 @@ -273,3 +281,12 @@ commands = dbt clean dbt deps dbt build --target postgres + +[testenv:integration_postgres_1_6_0] +changedir = integration_test_project +deps = dbt-postgres~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + From 5f25576c406f850d94042f34befa8c62fb0a0ebc Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 29 Sep 2023 15:22:39 +0100 Subject: [PATCH 22/23] Formatting --- integration_test_project/profiles.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index c33d5b3f..b24ad80d 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -2,8 +2,8 @@ # You should __NEVER__ check credentials into version control. Thanks for reading :) config: - send_anonymous_usage_stats: False - use_colors: True + send_anonymous_usage_stats: False + use_colors: True dbt_artifacts: target: snowflake @@ -52,4 +52,3 @@ dbt_artifacts: dbname: postgres schema: public threads: 8 - From a292b4585e96c0bb4619e850a64594f28fb96e91 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 29 Sep 2023 15:25:30 +0100 Subject: [PATCH 23/23] Update to include new features --- .../upload_exposures.sql | 25 +--- .../upload_invocations.sql | 121 +++++++----------- .../upload_model_executions.sql | 80 ++++-------- .../upload_models.sql | 25 +--- .../upload_seed_executions.sql | 20 --- .../upload_seeds.sql | 22 +--- .../upload_snapshot_executions.sql | 20 --- .../upload_snapshots.sql | 24 +--- .../upload_sources.sql | 22 +--- .../upload_test_executions.sql | 17 --- .../upload_tests.sql | 19 +-- .../insert_into_metadata_table.sql | 13 +- 12 files changed, 107 insertions(+), 301 deletions(-) diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index efeade2f..9f0ec5d3 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -88,23 +88,6 @@ {% macro postgres__get_exposures_dml_sql(exposures) -%} {% if exposures != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'name', - 'type', - 'owner', - 'maturity', - 'path', - 'description', - 'url', - 'package_name', - 'depends_on_nodes', - 'tags', - 'all_results', - ] %} - {% set exposure_values %} {% for exposure in exposures -%} ( @@ -121,13 +104,15 @@ '{{ exposure.package_name }}', {# package_name #} $${{ tojson(exposure.depends_on.nodes) }}$$, {# depends_on_nodes #} $${{ tojson(exposure.tags) }}$$, {# tags #} - $${{ tojson(exposure) }}$$ {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(exposure) }}$$ {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ exposure_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index 6d3d5c28..21c5574c 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -161,88 +161,63 @@ {% endmacro -%} {% macro postgres__get_invocations_dml_sql() -%} - {% set columns = [ - 'command_invocation_id', - 'dbt_version', - 'project_name', - 'run_started_at', - 'dbt_command', - 'full_refresh_flag', - 'target_profile_name', - 'target_name', - 'target_schema', - 'target_threads', - 'dbt_cloud_project_id', - 'dbt_cloud_job_id', - 'dbt_cloud_run_id', - 'dbt_cloud_run_reason_category', - 'dbt_cloud_run_reason', - 'env_vars', - 'dbt_vars', - 'invocation_args', - 'dbt_custom_envs', - ] %} - {% set invocation_values %} ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ dbt_version }}', {# dbt_version #} - '{{ project_name }}', {# project_name #} - '{{ run_started_at }}', {# run_started_at #} - '{{ flags.WHICH }}', {# dbt_command #} - {{ flags.FULL_REFRESH }}, {# full_refresh_flag #} - '{{ target.profile_name }}', {# target_profile_name #} - '{{ target.name }}', {# target_name #} - '{{ target.schema }}', {# target_schema #} - {{ target.threads }}, {# target_threads #} - - '{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #} - '{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #} - '{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #} - '{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #} - $${{ env_var('DBT_CLOUD_RUN_REASON', '') }}$$, {# dbt_cloud_run_reason #} - - {% if var('env_vars', none) %} - {% set env_vars_dict = {} %} - {% for env_variable in var('env_vars') %} - {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} - {% endfor %} - $${{ tojson(env_vars_dict) }}$$, {# env_vars #} - {% else %} - null, {# env_vars #} - {% endif %} - - {% if var('dbt_vars', none) %} - {% set dbt_vars_dict = {} %} - {% for dbt_var in var('dbt_vars') %} - {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} - {% endfor %} - $${{ tojson(dbt_vars_dict) }}$$, {# dbt_vars #} - {% else %} - null, {# dbt_vars #} - {% endif %} + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + {{ flags.FULL_REFRESH }}, {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var("DBT_CLOUD_PROJECT_ID", "") }}', {# dbt_cloud_project_id #} + '{{ env_var("DBT_CLOUD_JOB_ID", "") }}', {# dbt_cloud_job_id #} + '{{ env_var("DBT_CLOUD_RUN_ID", "") }}', {# dbt_cloud_run_id #} + '{{ env_var("DBT_CLOUD_RUN_REASON_CATEGORY", "") }}', {# dbt_cloud_run_reason_category #} + $${{ env_var('DBT_CLOUD_RUN_REASON', '') }}$$, {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} + {% endfor %} + $${{ tojson(env_vars_dict) }}$$, {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} - {% if invocation_args_dict.vars %} - {# vars - different format for pre v1.5 (yaml vs list) #} - {% if invocation_args_dict.vars is string %} - {# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #} - {% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %} - {% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %} + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} + {% endfor %} + $${{ tojson(dbt_vars_dict) }}$$, {# dbt_vars #} + {% else %} + null, {# dbt_vars #} {% endif %} - {% endif %} - $${{ tojson(invocation_args_dict) }}$$, {# invocation_args #} + {% if invocation_args_dict.vars %} + {# vars - different format for pre v1.5 (yaml vs list) #} + {% if invocation_args_dict.vars is string %} + {# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #} + {% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %} + {% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %} + {% endif %} + {% endif %} - {% set metadata_env = {} %} - {% for key, value in dbt_metadata_envs.items() %} - {% do metadata_env.update({key: value}) %} - {% endfor %} - $${{ tojson(metadata_env) }}$$ {# dbt_custom_envs #} + $${{ tojson(invocation_args_dict) }}$$, {# invocation_args #} + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: value}) %} + {% endfor %} + $${{ tojson(metadata_env) }}$$ {# dbt_custom_envs #} ) {% endset %} - {{ "(" ~ columns | join(', ') + ")" }} - VALUES {{ invocation_values }} {% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index c37fceea..bca26fea 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -165,75 +165,39 @@ {% macro postgres__get_model_executions_dml_sql(models) -%} {% if models != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'was_full_refresh', - 'thread_id', - 'status', - 'compile_started_at', - 'query_completed_at', - 'total_node_runtime', - 'rows_affected', - 'materialization', - 'schema', - 'name', - 'alias', - 'message', - 'adapter_response', - ]%} {% set model_execution_values %} {% for model in models -%} ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ model.node.unique_id }}', {# node_id #} - '{{ run_started_at }}', {# run_started_at #} + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} - {% set config_full_refresh = model.node.config.full_refresh %} - {% if config_full_refresh is none %} - {% set config_full_refresh = flags.FULL_REFRESH %} - {% endif %} - {{ config_full_refresh }}, {# was_full_refresh #} + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} - '{{ model.thread_id }}', {# thread_id #} - '{{ model.status }}', {# status #} + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} - {{ model.execution_time }}, {# total_node_runtime #} - null, - '{{ model.node.config.materialized }}', {# materialization #} - '{{ model.node.schema }}', {# schema #} - '{{ model.node.name }}', {# name #} - '{{ model.node.alias }}', {# alias #} - $${{ model.message }}$$, {# message #} - $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + {{ model.execution_time }}, {# total_node_runtime #} + null, {# rows_affected #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ model_execution_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index 805fde85..d8dc49fd 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -92,23 +92,6 @@ {% macro postgres__get_models_dml_sql(models) -%} {% if models != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'database', - 'schema', - 'name', - 'depends_on_nodes', - 'package_name', - 'path', - 'checksum', - 'materialization', - 'tags', - 'meta', - 'alias', - 'all_results', - ] %} {% set model_values %} {% for model in models -%} {% do model.pop('raw_code', None) %} @@ -127,13 +110,15 @@ '{{ tojson(model.tags) }}', {# tags #} $${{ model.config.meta }}$$, {# meta #} '{{ model.alias }}', {# alias #} - $${{ tojson(model) }}$$ {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(model) }}$$ {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ model_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index fd09dd87..ca947ac2 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -211,24 +211,6 @@ {% macro postgres__get_seed_executions_dml_sql(seeds) -%} {% if seeds != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'was_full_refresh', - 'thread_id', - 'status', - 'compile_started_at', - 'query_completed_at', - 'total_node_runtime', - 'rows_affected', - 'materialization', - 'schema', - 'name', - 'alias', - 'message', - 'adapter_response', - ]%} {% set seed_execution_values %} {% for model in seeds -%} ( @@ -278,8 +260,6 @@ {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ seed_execution_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 6b957864..32e67383 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -81,20 +81,6 @@ {% macro postgres__get_seeds_dml_sql(seeds) -%} {% if seeds != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'database', - 'schema', - 'name', - 'package_name', - 'path', - 'checksum', - 'meta', - 'alias', - 'all_results', - ] %} {% set seed_values %} {% for seed in seeds -%} ( @@ -109,13 +95,15 @@ '{{ seed.checksum.checksum }}', {# checksum #} $${{ tojson(seed.config.meta) }}$$, {# meta #} '{{ seed.alias }}', {# alias #} - $${{ tojson(seed) }}$$ {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(seed) }}$$ {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ seed_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index e1f11447..369348cd 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -211,24 +211,6 @@ {% macro postgres__get_snapshot_executions_dml_sql(snapshots) -%} {% if snapshots != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'was_full_refresh', - 'thread_id', - 'status', - 'compile_started_at', - 'query_completed_at', - 'total_node_runtime', - 'rows_affected', - 'materialization', - 'schema', - 'name', - 'alias', - 'message', - 'adapter_response', - ]%} {% set snapshot_execution_values %} {% for model in snapshots -%} ( @@ -278,8 +260,6 @@ {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ snapshot_execution_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index 54d01b25..0be6759b 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -89,22 +89,6 @@ {% macro postgres__get_snapshots_dml_sql(snapshots) -%} {% if snapshots != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'database', - 'schema', - 'name', - 'depends_on_nodes', - 'package_name', - 'path', - 'checksum', - 'strategy', - 'meta', - 'alias', - 'all_results', - ]%} {% set snapshot_values %} {% for snapshot in snapshots -%} ( @@ -121,13 +105,15 @@ '{{ snapshot.config.strategy }}', {# strategy #} $${{ tojson(snapshot.config.meta) }}$$, {# meta #} '{{ snapshot.alias }}', {# alias #} - $${{ tojson(snapshot) }}$$ {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(snapshot) }}$$ {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ snapshot_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index daa49c3c..dbcb49e4 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -81,20 +81,6 @@ {% macro postgres__get_sources_dml_sql(sources) -%} {% if sources != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'database', - 'schema', - 'source_name', - 'loader', - 'name', - 'identifier', - 'loaded_at_field', - 'freshness', - 'all_results', - ] %} {% set source_values %} {% for source in sources -%} ( @@ -109,13 +95,15 @@ '{{ source.identifier }}', {# identifier #} $${{ source.loaded_at_field }}$$, {# loaded_at_field #} $${{ tojson(source.freshness) }}$$, {# freshness #} - $${{ tojson(source) }}$$ {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(source) }}$$ {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ source_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index 910f37e9..a42b03ba 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -128,21 +128,6 @@ {% macro postgres__get_test_executions_dml_sql(tests) -%} {% if tests != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'was_full_refresh', - 'thread_id', - 'status', - 'compile_started_at', - 'query_completed_at', - 'total_node_runtime', - 'rows_affected', - 'failures', - 'message', - 'adapter_response', - ]%} {% set test_execution_values %} {% for test in tests -%} ( @@ -190,8 +175,6 @@ {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ test_execution_values }} {% else %} {{ return("") }} diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index 7d2fcfcf..cf75e9ba 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -72,17 +72,6 @@ {% macro postgres__get_tests_dml_sql(tests) -%} {% if tests != [] %} - {% set columns = [ - 'command_invocation_id', - 'node_id', - 'run_started_at', - 'name', - 'depends_on_nodes', - 'package_name', - 'test_path', - 'tags', - 'all_results', - ] %} {% set test_values %} {% for test in tests -%} ( @@ -94,13 +83,15 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} $${{ tojson(test.tags) }}$$, {# tags #} - $${{ tojson(test) }}$$ {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(test) }}$$ {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ "(" ~ columns | join(', ') ~ ")"}} - VALUES {{ test_values }} {% else %} {{ return("") }} diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index b1ad9993..24f1eb77 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -47,14 +47,15 @@ {%- endmacro %} -{% macro postgres__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} +{% macro postgres__insert_into_metadata_table(relation, fields, content) -%} - {% set insert_into_table_query %} - insert into {{ relation }} {{ fields }} - {{ content }} - {% endset %} + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + values + {{ content }} + {% endset %} - {% do run_query(insert_into_table_query) %} + {% do run_query(insert_into_table_query) %} {%- endmacro %}