diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 06c07e92..5eb359fd 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -113,7 +113,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest @@ -145,6 +145,12 @@ jobs: - name: Install tox run: python3 -m pip install tox + - name: Install SQL Server + run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=123" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2022-latest + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + - name: Checkout uses: actions/checkout@v3 with: diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 12a9c957..1c55194e 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest permissions: @@ -66,6 +66,12 @@ jobs: - name: Install tox run: python3 -m pip install tox + - name: Install SQL Server + run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=123" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2022-latest + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + - id: auth if: ${{ matrix.warehouse == 'bigquery' }} uses: google-github-actions/auth@v1 diff --git a/README.md b/README.md index 93c25f40..4569b8ed 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ The package currently supports - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: - Postgres :white_check_mark: +- SQL Server :white_check_mark: Models included: diff --git a/dbt_project.yml b/dbt_project.yml index 684efce6..4c05e3ae 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -15,8 +15,9 @@ models: +file_format: delta sources: +materialized: incremental - +on_schema_change: append_new_columns +full_refresh: false +persist_docs: - # Databricks doesn't offer column-level support for persisting docs - columns: '{{ target.name != "databricks" }}' + # Databricks and SQL Server don't offer column-level support for persisting docs + columns: '{{ target.name != "databricks" and target.type != "sqlserver" }}' + relation: '{{ target.type != "sqlserver" }}' + +as_columnstore: False diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index dba89a50..aae6f8df 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -25,8 +25,8 @@ vars: models: +persist_docs: - relation: true - columns: true + relation: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this + columns: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this seeds: +quote_columns: false diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index b24ad80d..33a1117a 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -52,3 +52,14 @@ dbt_artifacts: dbname: postgres schema: public threads: 8 + sqlserver: + type: sqlserver + driver: 'ODBC Driver 18 for SQL Server' + server: localhost + port: 1433 + database: dbt_artifact_integrationtests + schema: dbo + windows_login: False + trust_cert: True + user: sa + password: "123" diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql index a1e49e8c..35652754 100644 --- a/integration_test_project/tests/singular_test.sql +++ b/integration_test_project/tests/singular_test.sql @@ -1 +1,2 @@ -select 1 as failures from (select 2) as foo where 1 = 2 +select 1 as failures from (select 2 as two) as foo where 1 = 2 + diff --git a/macros/migration/migrate_from_v0_to_v1.sql b/macros/migration/migrate_from_v0_to_v1.sql index 63383077..b4cdc724 100644 --- a/macros/migration/migrate_from_v0_to_v1.sql +++ b/macros/migration/migrate_from_v0_to_v1.sql @@ -9,7 +9,7 @@ node_id, query_completed_at, rows_affected, - schema, + "schema", status, thread_id, total_node_runtime, @@ -34,9 +34,7 @@ {% endset %} {{ log("Migrating model_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_model_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_model_executions }} {%- endcall -%} {% set migrate_tests %} insert into {{new_database}}.{{new_schema}}.tests ( @@ -62,9 +60,7 @@ {% endset %} {{ log("Migrating tests", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_tests }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_tests }} {%- endcall -%} {% set migrate_test_executions %} insert into {{new_database}}.{{new_schema}}.test_executions ( @@ -96,22 +92,20 @@ {% endset %} {{ log("Migrating test_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_test_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_test_executions }} {%- endcall -%} {% set migrate_models %} insert into {{new_database}}.{{new_schema}}.models ( checksum, command_invocation_id, - database, + "database", depends_on_nodes, materialization, name, node_id, package_name, path, - schema, + "schema", run_started_at ) select @@ -130,20 +124,18 @@ {% endset %} {{ log("Migrating models", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_models }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_models }} {%- endcall -%} {% set migrate_seeds %} insert into {{new_database}}.{{new_schema}}.seeds ( checksum, command_invocation_id, - database, + "database", name, node_id, package_name, path, - schema, + "schema", run_started_at ) select @@ -160,9 +152,7 @@ {% endset %} {{ log("Migrating seeds", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_seeds }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_seeds }} {%- endcall -%} {% set migrate_seed_executions %} insert into {{new_database}}.{{new_schema}}.seed_executions ( @@ -173,7 +163,7 @@ node_id, query_completed_at, rows_affected, - schema, + "schema", status, thread_id, total_node_runtime, @@ -198,9 +188,7 @@ {% endset %} {{ log("Migrating seed_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_seed_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_seed_executions }} {%- endcall -%} {% set migrate_exposures %} insert into {{new_database}}.{{new_schema}}.exposures ( @@ -235,21 +223,19 @@ {% endset %} {{ log("Migrating exposures", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_exposures }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_exposures }} {%- endcall -%} {% set migrate_snapshots %} insert into {{new_database}}.{{new_schema}}.snapshots ( checksum, command_invocation_id, - database, + "database", depends_on_nodes, name, node_id, package_name, path, - schema, + "schema", strategy, run_started_at ) @@ -269,9 +255,7 @@ {% endset %} {{ log("Migrating snapshots", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_snapshots }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_snapshots }} {%- endcall -%} {% set migrate_snapshot_executions %} insert into {{new_database}}.{{new_schema}}.snapshot_executions ( @@ -282,7 +266,7 @@ node_id, query_completed_at, rows_affected, - schema, + "schema", status, thread_id, total_node_runtime, @@ -307,21 +291,19 @@ {% endset %} {{ log("Migrating snapshot_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_snapshot_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_snapshot_executions }} {%- endcall -%} {% set migrate_sources %} insert into {{new_database}}.{{new_schema}}.sources ( command_invocation_id, - database, + "database", freshness, identifier, loaded_at_field, loader, name, node_id, - schema, + "schema", source_name, run_started_at ) @@ -341,9 +323,8 @@ {% endset %} {{ log("Migrating sources", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_sources }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_sources }} {%- endcall -%} {{ log("Migration complete. You can now safely delete any data from before 1.0.0", info=True) }} {%- endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index 9f0ec5d3..e097be7e 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -1,5 +1,5 @@ {% macro upload_exposures(exposures) -%} - {{ return(adapter.dispatch('get_exposures_dml_sql', 'dbt_artifacts')(exposures)) }} + {{ return(adapter.dispatch("get_exposures_dml_sql", "dbt_artifacts")(exposures)) }} {%- endmacro %} {% macro default__get_exposures_dml_sql(exposures) -%} @@ -47,8 +47,7 @@ {%- endfor %} {% endset %} {{ exposure_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -80,8 +79,7 @@ {%- endfor %} {% endset %} {{ exposure_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -114,7 +112,45 @@ {%- endfor %} {% endset %} {{ exposure_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_exposures_dml_sql(exposures) -%} + + {% if exposures != [] %} + {% set exposure_values %} + select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14" + from ( values + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ exposure.unique_id | replace("'","''") }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ exposure.name | replace("'","''") }}', {# name #} + '{{ exposure.type }}', {# type #} + '{{ tojson(exposure.owner) }}', {# owner #} + '{{ exposure.maturity }}', {# maturity #} + '{{ exposure.original_file_path }}', {# path #} + '{{ exposure.description | replace("'","''") }}', {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ tojson(exposure.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(exposure) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14") + + {% endset %} + {{ exposure_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index 21c5574c..775861ac 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -2,24 +2,22 @@ {# Need to remove keys with results that can't be handled properly #} {# warn_error_options - returns a python object in 1.5 #} - {% if 'warn_error_options' in invocation_args_dict %} + {% if "warn_error_options" in invocation_args_dict %} {% if invocation_args_dict.warn_error_options is not string %} {% if invocation_args_dict.warn_error_options.include %} {% set include_options = invocation_args_dict.warn_error_options.include %} - {% else %} - {% set include_options = '' %} + {% else %} {% set include_options = "" %} {% endif %} {% if invocation_args_dict.warn_error_options.exclude %} {% set exclude_options = invocation_args_dict.warn_error_options.exclude %} - {% else %} - {% set exclude_options = '' %} + {% else %} {% set exclude_options = "" %} {% endif %} - {% set warn_error_options = {'include': include_options, 'exclude': exclude_options} %} - {%- do invocation_args_dict.update({'warn_error_options': warn_error_options}) %} + {% set warn_error_options = {"include": include_options, "exclude": exclude_options} %} + {%- do invocation_args_dict.update({"warn_error_options": warn_error_options}) %} {% endif %} {% endif %} - {{ return(adapter.dispatch('get_invocations_dml_sql', 'dbt_artifacts')()) }} + {{ return(adapter.dispatch("get_invocations_dml_sql", "dbt_artifacts")()) }} {%- endmacro %} {% macro default__get_invocations_dml_sql() -%} @@ -221,3 +219,80 @@ {{ invocation_values }} {% endmacro -%} + + +{% macro sqlserver__get_invocations_dml_sql() -%} + {% set invocation_values %} + select + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + nullif("11", ''), + nullif("12", ''), + nullif("13", ''), + nullif("14", ''), + nullif("15", ''), + "16", + "17", + "18", + "19" + from (values + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + '{{ flags.FULL_REFRESH }}', {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #} + '{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #} + '{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #} + '{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #} + '{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","''") }}', {# dbt_cloud_run_reason #} + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, '') | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(env_vars_dict) }}', {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, '') | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(dbt_vars_dict) }}', {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + '{{ tojson(invocation_args_dict) | replace("'", "''") }}', {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: (value | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(metadata_env) }}' {# dbt_custom_envs #} + + ) + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19") + + {% endset %} + {{ invocation_values }} + +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index bca26fea..1d5fa83b 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -1,5 +1,5 @@ {% macro upload_model_executions(models) -%} - {{ return(adapter.dispatch('get_model_executions_dml_sql', 'dbt_artifacts')(models)) }} + {{ return(adapter.dispatch("get_model_executions_dml_sql", "dbt_artifacts")(models)) }} {%- endmacro %} {% macro default__get_model_executions_dml_sql(models) -%} @@ -57,8 +57,7 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -99,8 +98,7 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -158,8 +156,7 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -199,7 +196,53 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) v ( "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" ) + + {% endset %} + {{ model_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index 7570b06a..2c27daec 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -1,5 +1,5 @@ {% macro upload_models(models) -%} - {{ return(adapter.dispatch('get_models_dml_sql', 'dbt_artifacts')(models)) }} + {{ return(adapter.dispatch("get_models_dml_sql", "dbt_artifacts")(models)) }} {%- endmacro %} {% macro default__get_models_dml_sql(models) -%} @@ -51,8 +51,7 @@ {%- endfor %} {% endset %} {{ model_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -87,8 +86,7 @@ {%- endfor %} {% endset %} {{ model_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -123,7 +121,50 @@ {%- endfor %} {% endset %} {{ model_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_models_dml_sql(models) -%} + + {% if models != [] %} + {% set model_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15" + from ( values + {% for model in models -%} + {% set model_copy = model.copy() -%} + {% do model_copy.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model_copy.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ model_copy.database }}', {# database #} + '{{ model_copy.schema }}', {# schema #} + '{{ model_copy.name }}', {# name #} + '{{ tojson(model_copy.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ model_copy.package_name }}', {# package_name #} + '{{ model_copy.original_file_path }}', {# path #} + '{{ model_copy.checksum.checksum }}', {# checksum #} + '{{ model_copy.config.materialized }}', {# materialization #} + '{{ tojson(model_copy.tags) }}', {# tags #} + '{{ tojson(model_copy.config.meta) | replace("'","''") }}', {# meta #} + '{{ model_copy.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(model_copy) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15") + + {% endset %} + {{ model_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 1ccbfe2a..c19a9a2f 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -1,5 +1,5 @@ {% macro upload_seed_executions(seeds) -%} - {{ return(adapter.dispatch('get_seed_executions_dml_sql', 'dbt_artifacts')(seeds)) }} + {{ return(adapter.dispatch("get_seed_executions_dml_sql", "dbt_artifacts")(seeds)) }} {%- endmacro %} {% macro default__get_seed_executions_dml_sql(seeds) -%} @@ -56,8 +56,7 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -97,8 +96,7 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -156,8 +154,7 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -213,7 +210,53 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16") + + {% endset %} + {{ seed_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 32e67383..2501a9e9 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -1,5 +1,5 @@ {% macro upload_seeds(seeds) -%} - {{ return(adapter.dispatch('get_seeds_dml_sql', 'dbt_artifacts')(seeds)) }} + {{ return(adapter.dispatch("get_seeds_dml_sql", "dbt_artifacts")(seeds)) }} {%- endmacro %} {% macro default__get_seeds_dml_sql(seeds) -%} @@ -43,8 +43,7 @@ {%- endfor %} {% endset %} {{ seed_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -74,8 +73,7 @@ {%- endfor %} {% endset %} {{ seed_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -105,7 +103,43 @@ {%- endfor %} {% endset %} {{ seed_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_seeds_dml_sql(seeds) -%} + + {% if seeds != [] %} + {% set seed_values %} + select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" + from ( values + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + '{{ tojson(seed.config.meta) | replace("'","''") }}', {# meta #} + '{{ seed.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(seed) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12") + + {% endset %} + {{ seed_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index 2006b168..b0af313b 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -1,5 +1,5 @@ {% macro upload_snapshot_executions(snapshots) -%} - {{ return(adapter.dispatch('get_snapshot_executions_dml_sql', 'dbt_artifacts')(snapshots)) }} + {{ return(adapter.dispatch("get_snapshot_executions_dml_sql", "dbt_artifacts")(snapshots)) }} {%- endmacro %} {% macro default__get_snapshot_executions_dml_sql(snapshots) -%} @@ -56,8 +56,7 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -97,8 +96,7 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -156,8 +154,7 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -213,7 +210,53 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16") + + {% endset %} + {{ snapshot_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index 0be6759b..e540dc0b 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -1,6 +1,6 @@ {% macro upload_snapshots(snapshots) -%} - {{ return(adapter.dispatch('get_snapshots_dml_sql', 'dbt_artifacts')(snapshots)) }} + {{ return(adapter.dispatch("get_snapshots_dml_sql", "dbt_artifacts")(snapshots)) }} {%- endmacro %} @@ -49,8 +49,7 @@ {%- endfor %} {% endset %} {{ snapshot_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -82,8 +81,7 @@ {%- endfor %} {% endset %} {{ snapshot_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -115,7 +113,47 @@ {%- endfor %} {% endset %} {{ snapshot_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_snapshots_dml_sql(snapshots) -%} + + {% if snapshots != [] %} + {% set snapshot_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14" + from ( values + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + '{{ tojson(snapshot.config.meta) | replace("'","''") }}', {# meta #} + '{{ snapshot.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(snapshot) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14") + + {% endset %} + {{ snapshot_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index dbcb49e4..3d899755 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -1,5 +1,5 @@ {% macro upload_sources(sources) -%} - {{ return(adapter.dispatch('get_sources_dml_sql', 'dbt_artifacts')(sources)) }} + {{ return(adapter.dispatch("get_sources_dml_sql", "dbt_artifacts")(sources)) }} {%- endmacro %} {% macro default__get_sources_dml_sql(sources) -%} @@ -43,8 +43,7 @@ {%- endfor %} {% endset %} {{ source_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -74,8 +73,7 @@ {%- endfor %} {% endset %} {{ source_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -105,7 +103,43 @@ {%- endfor %} {% endset %} {{ source_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_sources_dml_sql(sources) -%} + + {% if sources != [] %} + {% set source_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" + from ( values + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + '{{ source.loaded_at_field | replace("'","''") }}', {# loaded_at_field #} + '{{ tojson(source.freshness) | replace("'","''") }}', {# freshness #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(source) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12") + {% endset %} + {{ source_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index cb13288b..3af31d73 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -1,5 +1,5 @@ {% macro upload_test_executions(tests) -%} - {{ return(adapter.dispatch('get_test_executions_dml_sql', 'dbt_artifacts')(tests)) }} + {{ return(adapter.dispatch("get_test_executions_dml_sql", "dbt_artifacts")(tests)) }} {%- endmacro %} {% macro default__get_test_executions_dml_sql(tests) -%} @@ -50,8 +50,7 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -89,8 +88,7 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -144,8 +142,7 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -197,7 +194,51 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} -{% endmacro -%} \ No newline at end of file +{% endmacro -%} + +{% macro sqlserver__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13" + from ( values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ test.execution_time }}, {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + '{{ test.message | replace("'", "''") }}', {# message #} + '{{ tojson(test.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) AS v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13") + + {% endset %} + {{ test_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + + diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index cf75e9ba..18d7171c 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -1,5 +1,5 @@ {% macro upload_tests(tests) -%} - {{ return(adapter.dispatch('get_tests_dml_sql', 'dbt_artifacts')(tests)) }} + {{ return(adapter.dispatch("get_tests_dml_sql", "dbt_artifacts")(tests)) }} {%- endmacro %} {% macro default__get_tests_dml_sql(tests) -%} @@ -37,8 +37,7 @@ {%- endfor %} {% endset %} {{ test_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -65,8 +64,7 @@ {%- endfor %} {% endset %} {{ test_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -93,7 +91,40 @@ {%- endfor %} {% endset %} {{ test_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_tests_dml_sql(tests) -%} + + {% if tests != [] %} + {% set test_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9" + from ( values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path }}', {# test_path #} + '{{ tojson(test.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(test) | replace("'","''") }}' {# all_fields #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9") + {% endset %} + {{ test_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_results/get_column_name_lists.sql b/macros/upload_results/get_column_name_lists.sql index 7911e866..707e0c2c 100644 --- a/macros/upload_results/get_column_name_lists.sql +++ b/macros/upload_results/get_column_name_lists.sql @@ -1,12 +1,10 @@ - {# These are the column lists used as part of the upload macros - the order here should be the same as the order in each individual `upload_dataset` macro. #} - {% macro get_column_name_list(dataset) -%} - {% if dataset == 'exposures' %} + {% if dataset == "exposures" %} ( command_invocation_id, @@ -25,7 +23,7 @@ all_results ) - {% elif dataset == 'invocations' %} + {% elif dataset == "invocations" %} ( command_invocation_id, @@ -49,7 +47,7 @@ dbt_custom_envs ) - {% elif dataset == 'model_executions' %} + {% elif dataset == "model_executions" %} ( command_invocation_id, @@ -62,25 +60,27 @@ query_completed_at, total_node_runtime, rows_affected, - {% if target.type == 'bigquery' %} - bytes_processed, + {% if target.type == "bigquery" %} bytes_processed, {% endif %} materialization, - schema, + {% if target.type == "sqlserver" %} "schema", + {% else %} schema, + {% endif %} name, alias, message, adapter_response ) - {% elif dataset == 'models' %} + {% elif dataset == "models" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} name, depends_on_nodes, package_name, @@ -93,8 +93,7 @@ all_results ) - - {% elif dataset == 'seed_executions' %} + {% elif dataset == "seed_executions" %} ( command_invocation_id, @@ -108,21 +107,24 @@ total_node_runtime, rows_affected, materialization, - schema, + {% if target.type == "sqlserver" %} "schema", + {% else %} schema, + {% endif %} name, alias, message, adapter_response ) - {% elif dataset == 'seeds' %} + {% elif dataset == "seeds" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} name, package_name, path, @@ -132,7 +134,7 @@ all_results ) - {% elif dataset == 'snapshot_executions' %} + {% elif dataset == "snapshot_executions" %} ( command_invocation_id, @@ -146,21 +148,24 @@ total_node_runtime, rows_affected, materialization, - schema, + {% if target.type == "sqlserver" %} "schema", + {% else %} schema, + {% endif %} name, alias, message, adapter_response ) - {% elif dataset == 'snapshots' %} + {% elif dataset == "snapshots" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} name, depends_on_nodes, package_name, @@ -172,14 +177,15 @@ all_results ) - {% elif dataset == 'sources' %} + {% elif dataset == "sources" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} source_name, loader, name, @@ -189,7 +195,7 @@ all_results ) - {% elif dataset == 'test_executions' %} + {% elif dataset == "test_executions" %} ( command_invocation_id, @@ -207,7 +213,7 @@ adapter_response ) - {% elif dataset == 'tests' %} + {% elif dataset == "tests" %} ( command_invocation_id, @@ -223,8 +229,8 @@ {% else %} - /* No column list available */ - + /* No column list available */ {% endif %} {%- endmacro %} + diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 24f1eb77..8d4afa40 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -59,5 +59,17 @@ {%- endmacro %} +{% macro sqlserver__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} + diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 3e24ae79..4e6ceb1b 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,108 +1,135 @@ with - base as ( - - select * from {{ ref('stg_dbt__models') }} - - ) - - , model_executions as ( - - select * from {{ ref('stg_dbt__model_executions') }} - - ) - - , latest_models as ( + base as (select * from {{ ref("stg_dbt__models") }}), + model_executions as (select * from {{ ref("stg_dbt__model_executions") }}), + latest_models as ( /* Retrieves the models present in the most recent run */ - select * - from base - where run_started_at = (select max(run_started_at) from base) + select * from base where run_started_at = (select max(run_started_at) from base) - ) + ), - , latest_models_runs as ( + latest_models_runs as ( /* Retreives all successful run information for the models present in the most recent run and ranks them based on query completion time */ select - model_executions.node_id - , model_executions.was_full_refresh - , model_executions.query_completed_at - , model_executions.total_node_runtime - , model_executions.rows_affected - {% if target.type == 'bigquery' %} - , model_executions.bytes_processed - {% endif %} + model_executions.node_id, + model_executions.was_full_refresh, + model_executions.query_completed_at, + model_executions.total_node_runtime, + model_executions.rows_affected + {% if target.type == "bigquery" %}, model_executions.bytes_processed {% endif %}, /* Row number by refresh and node ID */ - , row_number() over ( + row_number() over ( partition by latest_models.node_id, model_executions.was_full_refresh - order by model_executions.query_completed_at desc /* most recent ranked first */ - ) as run_idx + order by model_executions.query_completed_at desc /* most recent ranked first */ + ) as run_idx, /* Row number by node ID */ - , row_number() over ( - partition by latest_models.node_id - order by model_executions.query_completed_at desc /* most recent ranked first */ + row_number() over ( + partition by latest_models.node_id order by model_executions.query_completed_at desc /* most recent ranked first */ ) as run_idx_id_only from model_executions inner join latest_models on model_executions.node_id = latest_models.node_id where model_executions.status = 'success' - ) + ), - , latest_model_stats as ( + latest_model_stats as ( select - node_id - , max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at - , max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime - , max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed - {% endif %} - , max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at - , max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime - , max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed - {% endif %} - , max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at - , max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime - , max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed + node_id, + max( + case + when was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then query_completed_at + end + ) as last_full_refresh_run_completed_at, + max( + case + when was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then total_node_runtime + end + ) as last_full_refresh_run_total_runtime, + max( + case + when was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then rows_affected + end + ) as last_full_refresh_run_rows_affected + {% if target.type == "bigquery" %} + , + max( + case when was_full_refresh then bytes_processed end + ) as last_full_refresh_run_bytes_processed + {% endif %}, + max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at, + max( + case when run_idx_id_only = 1 then total_node_runtime end + ) as last_run_total_runtime, + max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected + {% if target.type == "bigquery" %} + , + max( + case when run_idx_id_only = 1 then bytes_processed end + ) as last_run_bytes_processed + {% endif %}, + max( + case + when not was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then query_completed_at + end + ) as last_incremental_run_completed_at, + max( + case + when not was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then total_node_runtime + end + ) as last_incremental_run_total_runtime, + max( + case + when not was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then rows_affected + end + ) as last_incremental_run_rows_affected + {% if target.type == "bigquery" %} + , + max( + case when not was_full_refresh then bytes_processed end + ) as last_incremental_run_bytes_processed {% endif %} from latest_models_runs where run_idx = 1 - group by 1 + group by node_id - ) + ), - , final as ( + final as ( select - latest_models.* - , latest_model_stats.last_full_refresh_run_completed_at - , latest_model_stats.last_full_refresh_run_total_runtime - , latest_model_stats.last_full_refresh_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_full_refresh_run_bytes_processed - {% endif %} - , latest_model_stats.last_run_completed_at - , latest_model_stats.last_run_total_runtime - , latest_model_stats.last_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_run_bytes_processed - {% endif %} - , latest_model_stats.last_incremental_run_completed_at - , latest_model_stats.last_incremental_run_total_runtime - , latest_model_stats.last_incremental_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_incremental_run_bytes_processed + latest_models.*, + latest_model_stats.last_full_refresh_run_completed_at, + latest_model_stats.last_full_refresh_run_total_runtime, + latest_model_stats.last_full_refresh_run_rows_affected + {% if target.type == "bigquery" %} + , latest_model_stats.last_full_refresh_run_bytes_processed + {% endif %}, + latest_model_stats.last_run_completed_at, + latest_model_stats.last_run_total_runtime, + latest_model_stats.last_run_rows_affected + {% if target.type == "bigquery" %} + , latest_model_stats.last_run_bytes_processed + {% endif %}, + latest_model_stats.last_incremental_run_completed_at, + latest_model_stats.last_incremental_run_total_runtime, + latest_model_stats.last_incremental_run_rows_affected + {% if target.type == "bigquery" %} + , latest_model_stats.last_incremental_run_bytes_processed {% endif %} from latest_models - left join latest_model_stats - on latest_models.node_id = latest_model_stats.node_id + left join latest_model_stats on latest_models.node_id = latest_model_stats.node_id ) -select * from final +select * +from final + diff --git a/models/dim_dbt__models.sql b/models/dim_dbt__models.sql index 8ed90ce0..34f066a2 100644 --- a/models/dim_dbt__models.sql +++ b/models/dim_dbt__models.sql @@ -1,31 +1,32 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__models") }}), - select * - from {{ ref('stg_dbt__models') }} - - ) - - , models as ( + models as ( select - model_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , depends_on_nodes - , package_name - , path - , checksum - , materialization - , tags - , meta - , alias + model_execution_id, + command_invocation_id, + node_id, + run_started_at, + name, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + depends_on_nodes, + package_name, + path, + checksum, + materialization, + tags, + meta, + alias from base ) -select * from models +select * +from models + diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index d07c288d..cf005fbf 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -1,28 +1,29 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__seeds") }}), - select * - from {{ ref('stg_dbt__seeds') }} - - ) - - , seeds as ( + seeds as ( select - seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , package_name - , path - , checksum - , meta - , alias + seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + name, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + package_name, + path, + checksum, + meta, + alias from base ) -select * from seeds +select * +from seeds + diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index a3270714..d1d0a999 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -1,30 +1,31 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__snapshots") }}), - select * - from {{ ref('stg_dbt__snapshots') }} - - ) - - , snapshots as ( + snapshots as ( select - snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , depends_on_nodes - , package_name - , path - , checksum - , strategy - , meta - , alias + snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + name, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + depends_on_nodes, + package_name, + path, + checksum, + strategy, + meta, + alias from base ) -select * from snapshots +select * +from snapshots + diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index affe5b99..c6e3263a 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -1,28 +1,29 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__sources") }}), - select * - from {{ ref('stg_dbt__sources') }} - - ) - - , sources as ( + sources as ( select - source_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , source_name - , loader - , name - , identifier - , loaded_at_field - , freshness + source_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + source_name, + loader, + name, + identifier, + loaded_at_field, + freshness from base ) -select * from sources +select * +from sources + diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql index c6c93548..3b7c2368 100644 --- a/models/fct_dbt__model_executions.sql +++ b/models/fct_dbt__model_executions.sql @@ -1,35 +1,32 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__model_executions") }}), - select * - from {{ ref('stg_dbt__model_executions') }} - - ) - - , model_executions as ( + model_executions as ( select - model_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - {% if target.type == 'bigquery' %} - , bytes_processed - {% endif %} - , materialization - , schema -- noqa - , name - , alias - , message + model_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected + {% if target.type == "bigquery" %}, bytes_processed {% endif %}, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + name, + alias, + message from base ) -select * from model_executions +select * +from model_executions + diff --git a/models/fct_dbt__seed_executions.sql b/models/fct_dbt__seed_executions.sql index 39d2abb5..1e6465a0 100644 --- a/models/fct_dbt__seed_executions.sql +++ b/models/fct_dbt__seed_executions.sql @@ -1,32 +1,31 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__seed_executions") }}), - select * - from {{ ref('stg_dbt__seed_executions') }} - - ) - - , seed_executions as ( + seed_executions as ( select - seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , schema - , name - , alias - , message + seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + name, + alias, + message from base ) -select * from seed_executions +select * +from seed_executions + diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index 91613db0..58ca4408 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -1,32 +1,31 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__snapshot_executions") }}), - select * - from {{ ref('stg_dbt__snapshot_executions') }} - - ) - - , snapshot_executions as ( + snapshot_executions as ( select - snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , schema - , name - , alias - , message + snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + name, + alias, + message from base ) -select * from snapshot_executions +select * +from snapshot_executions + diff --git a/models/fct_dbt__test_executions.sql b/models/fct_dbt__test_executions.sql index 43b87d33..c9e347f0 100644 --- a/models/fct_dbt__test_executions.sql +++ b/models/fct_dbt__test_executions.sql @@ -1,29 +1,26 @@ with - base as ( + base as (select * from {{ ref("stg_dbt__test_executions") }}), - select * - from {{ ref('stg_dbt__test_executions') }} - - ) - - , test_executions as ( + test_executions as ( select - test_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , failures - , message + test_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + failures, + message from base ) -select * from test_executions +select * +from test_executions + diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index 14d6c6b2..5710affd 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -1,30 +1,28 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_boolean() }}) as was_full_refresh - , cast(null as {{ type_string() }}) as thread_id - , cast(null as {{ type_string() }}) as status - , cast(null as {{ type_timestamp() }}) as compile_started_at - , cast(null as {{ type_timestamp() }}) as query_completed_at - , cast(null as {{ type_float() }}) as total_node_runtime - , cast(null as {{ type_int() }}) as rows_affected - {% if target.type == 'bigquery' %} - , cast(null as {{ type_int() }}) as bytes_processed + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + {% if target.type == "bigquery" %} + cast(null as {{ type_int() }}) as bytes_processed, {% endif %} - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as schema - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_string() }}) as message - , cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_string() }}) as message, + cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 + diff --git a/models/sources/models.sql b/models/sources/models.sql index debeaf05..7ac8ee84 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -1,26 +1,26 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_array() }}) as depends_on_nodes - , cast(null as {{ type_string() }}) as package_name - , cast(null as {{ type_string() }}) as path - , cast(null as {{ type_string() }}) as checksum - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_array() }}) as tags - , cast(null as {{ type_json() }}) as meta - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_array() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_array() }}) as tags, + cast(null as {{ type_json() }}) as meta, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql index c3e9465d..95a750e6 100644 --- a/models/sources/seed_executions.sql +++ b/models/sources/seed_executions.sql @@ -1,27 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_boolean() }}) as was_full_refresh - , cast(null as {{ type_string() }}) as thread_id - , cast(null as {{ type_string() }}) as status - , cast(null as {{ type_timestamp() }}) as compile_started_at - , cast(null as {{ type_timestamp() }}) as query_completed_at - , cast(null as {{ type_float() }}) as total_node_runtime - , cast(null as {{ type_int() }}) as rows_affected - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as schema - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_string() }}) as message - , cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_string() }}) as message, + cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 + diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql index 3c99ef01..882c661a 100644 --- a/models/sources/seeds.sql +++ b/models/sources/seeds.sql @@ -1,23 +1,23 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as package_name - , cast(null as {{ type_string() }}) as path - , cast(null as {{ type_string() }}) as checksum - , cast(null as {{ type_json() }}) as meta - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_json() }}) as meta, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + diff --git a/models/sources/snapshot_executions.sql b/models/sources/snapshot_executions.sql index c3e9465d..95a750e6 100644 --- a/models/sources/snapshot_executions.sql +++ b/models/sources/snapshot_executions.sql @@ -1,27 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_boolean() }}) as was_full_refresh - , cast(null as {{ type_string() }}) as thread_id - , cast(null as {{ type_string() }}) as status - , cast(null as {{ type_timestamp() }}) as compile_started_at - , cast(null as {{ type_timestamp() }}) as query_completed_at - , cast(null as {{ type_float() }}) as total_node_runtime - , cast(null as {{ type_int() }}) as rows_affected - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as schema - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_string() }}) as message - , cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_string() }}) as message, + cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 + diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index c47ef812..77061731 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -1,24 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_array() }}) as depends_on_nodes - , cast(null as {{ type_string() }}) as package_name - , cast(null as {{ type_string() }}) as path - , cast(null as {{ type_string() }}) as checksum - , cast(null as {{ type_string() }}) as strategy - , cast(null as {{ type_json() }}) as meta - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_array() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_string() }}) as strategy, + cast(null as {{ type_json() }}) as meta, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + diff --git a/models/sources/sources.sql b/models/sources/sources.sql index d152ad22..91c8d1d3 100644 --- a/models/sources/sources.sql +++ b/models/sources/sources.sql @@ -1,25 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - select 1 as foo - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema - , cast(null as {{ type_string() }}) as source_name - , cast(null as {{ type_string() }}) as loader - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as identifier - , cast(null as {{ type_string() }}) as loaded_at_field - {% if target.type == 'snowflake' %} - , cast(null as {{ type_array() }}) as freshness - {% else %} - , cast(null as {{ type_json() }}) as freshness - {% endif %} - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as source_name, + cast(null as {{ type_string() }}) as loader, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as identifier, + cast(null as {{ type_string() }}) as loaded_at_field + {% if target.type == "snowflake" %}, cast(null as {{ type_array() }}) as freshness + {% else %}, cast(null as {{ type_json() }}) as freshness + {% endif %}, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 1381f6c4..ad2c4f50 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -1,36 +1,33 @@ with - base as ( - - select * - from {{ ref('model_executions') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("model_executions") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , {{ split_part('thread_id', "'-'", 2) }} as thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - {% if target.type == 'bigquery' %} - , bytes_processed - {% endif %} - , materialization - , schema -- noqa - , name - , alias - , message - , adapter_response + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as model_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + {{ split_part("thread_id", "'-'", 2) }} as thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected + {% if target.type == "bigquery" %}, bytes_processed {% endif %}, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias, + message, + adapter_response from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index 0af3b491..d8184183 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -1,31 +1,32 @@ with - base as ( - - select * - from {{ ref('models') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("models") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , depends_on_nodes - , package_name - , path - , checksum - , materialization - , tags - , meta - , alias + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as model_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + depends_on_nodes, + package_name, + path, + checksum, + materialization, + tags, + meta, + alias from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index ec90aaf0..90f52870 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -1,33 +1,32 @@ with - base as ( - - select * - from {{ ref('seed_executions') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("seed_executions") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , {{ split_part('thread_id', "'-'", 2) }} as thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , schema -- noqa - , name - , alias - , message - , adapter_response + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + {{ split_part("thread_id", "'-'", 2) }} as thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias, + message, + adapter_response from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 6e652631..0e5dcb1d 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -1,28 +1,29 @@ with - base as ( - - select * - from {{ ref('seeds') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("seeds") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , package_name - , path - , checksum - , meta - , alias + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + name, + package_name, + path, + checksum, + meta, + alias from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 5818133d..8794b99b 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -1,33 +1,32 @@ with - base as ( - - select * - from {{ ref('snapshot_executions') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("snapshot_executions") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , {{ split_part('thread_id', "'-'", 2) }} as thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , schema -- noqa - , name - , alias - , message - , adapter_response + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + {{ split_part("thread_id", "'-'", 2) }} as thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias, + message, + adapter_response from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index 7571e14b..8b2a66a4 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -1,30 +1,32 @@ with - base as ( + base as (select * from {{ ref("snapshots") }}), - select * - from {{ ref('snapshots') }} - - ) - - , enhanced as ( + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , depends_on_nodes - , package_name - , path - , checksum - , strategy - , meta - , alias + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + name, + depends_on_nodes, + package_name, + path, + checksum, + strategy, + meta, + alias from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index aca60a43..1bd8cef1 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -1,28 +1,29 @@ with - base as ( - - select * - from {{ ref('sources') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("sources") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as source_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , source_name - , loader - , name - , identifier - , loaded_at_field - , freshness + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as source_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + source_name, + loader, + name, + identifier, + loaded_at_field, + freshness from base ) -select * from enhanced +select * +from enhanced + diff --git a/tox.ini b/tox.ini index aee9fc2b..50e79ce7 100644 --- a/tox.ini +++ b/tox.ini @@ -353,4 +353,34 @@ commands = dbt deps dbt build --target postgres +[testenv:integration_sqlserver] +changedir = integration_test_project +deps = dbt-sqlserver~=1.8.4 +commands = + dbt clean + dbt deps + dbt build --target sqlserver + +[testenv:integration_sqlserver_1_4_3] +changedir = integration_test_project +deps = dbt-sqlserver~=1.4.3 +commands = + dbt clean + dbt deps + dbt build --target sqlserver +[testenv:integration_sqlserver_1_7_4] +changedir = integration_test_project +deps = dbt-sqlserver~=1.7.4 +commands = + dbt clean + dbt deps + dbt build --target sqlserver + +[testenv:integration_sqlserver_1_8_4] +changedir = integration_test_project +deps = dbt-sqlserver~=1.8.4 +commands = + dbt clean + dbt deps + dbt build --target sqlserver