diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 034a316f..f6fc13a8 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["snowflake", "bigquery", "postgres", "trino"] runs-on: ubuntu-latest environment: name: Approve Integration Tests @@ -58,6 +58,11 @@ jobs: ports: - 5432:5432 + trino: + image: trinodb/trino + ports: + - 8080:8080 + steps: - name: Get latest release uses: rez0n/actions-github-release@main @@ -113,7 +118,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["snowflake", "bigquery", "postgres", "trino"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0"] runs-on: ubuntu-latest @@ -136,6 +141,11 @@ jobs: ports: - 5432:5432 + trino: + image: trinodb/trino + ports: + - 8080:8080 + steps: - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index c8e503b5..4341e074 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["snowflake", "bigquery", "postgres", "trino"] version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0"] runs-on: ubuntu-latest permissions: @@ -54,6 +54,11 @@ jobs: ports: - 5432:5432 + trino: + image: trinodb/trino + ports: + - 8080:8080 + steps: - name: Checkout uses: actions/checkout@v3 diff --git a/README.md b/README.md index 6350d743..9027d61c 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ The package currently supports - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: - Postgres :white_check_mark: +- Trino :white_check_mark: Models included: diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index b24ad80d..2ad04547 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -52,3 +52,13 @@ dbt_artifacts: dbname: postgres schema: public threads: 8 + trino: + type: trino + method: none + user: trino + password: password + host: localhost + database: memory + schema: default + port: 8080 + threads: 8 \ No newline at end of file diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index 4064ad46..582e1879 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -43,3 +43,8 @@ {% macro bigquery__type_array() %} array {% endmacro %} + +{% macro trino__type_array() %} + array(varchar) +{% endmacro %} + diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index 9f0ec5d3..7d34f4bc 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -118,3 +118,38 @@ {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro trino__get_exposures_dml_sql(exposures) -%} + {% if exposures != [] %} + + {% set exposure_values %} + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ exposure.unique_id | replace("'","''") }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + '{{ exposure.name | replace("'","''") }}', {# name #} + '{{ exposure.type }}', {# type #} + '{{ tojson(exposure.owner) | replace("'","''") }}', {# owner #} + '{{ exposure.maturity }}', {# maturity #} + '{{ exposure.original_file_path }}', {# path #} + '{{ exposure.description | replace("'","''") }}', {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + ARRAY {{ exposure.depends_on.nodes}}, {# depends_on_nodes #} + ARRAY {{ exposure.tags}}, {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(exposure) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index 21c5574c..db166bdd 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -219,5 +219,66 @@ ) {% endset %} {{ invocation_values }} + +{% endmacro -%} + +{% macro trino__get_invocations_dml_sql() -%} + {% set invocation_values %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + {{ flags.FULL_REFRESH }}, {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var("DBT_CLOUD_PROJECT_ID", "") }}', {# dbt_cloud_project_id #} + '{{ env_var("DBT_CLOUD_JOB_ID", "") }}', {# dbt_cloud_job_id #} + '{{ env_var("DBT_CLOUD_RUN_ID", "") }}', {# dbt_cloud_run_id #} + '{{ env_var("DBT_CLOUD_RUN_REASON_CATEGORY", "") }}', {# dbt_cloud_run_reason_category #} + '{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","''") }}', {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} + {% endfor %} + '{{ tojson(env_vars_dict) | replace("'","''") }}', {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} + {% endfor %} + '{{ tojson(dbt_vars_dict) | replace("'","''") }}', {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + + {% if invocation_args_dict.vars %} + {# vars - different format for pre v1.5 (yaml vs list) #} + {% if invocation_args_dict.vars is string %} + {% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %} + {% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %} + {% endif %} + {% endif %} + + '{{ invocation_args_dict | replace("'","''") }}', {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: value}) %} + {% endfor %} + '{{ tojson(metadata_env) | replace("'","''") }}' {# dbt_custom_envs #} + ) + {% endset %} + {{ invocation_values }} {% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index bca26fea..6e5d2d0b 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -203,3 +203,51 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro trino__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + + {{ config_full_refresh }}, {# was_full_refresh #} + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}TIMESTAMP '{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}TIMESTAMP '{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + + {% if model.adapter_response.rows_affected is none %} + null + {% else %} + {{ model.adapter_response.rows_affected }} + {% endif %} + , {# rows_affected #} + + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index 7570b06a..eb7f3d01 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -127,3 +127,38 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro trino__get_models_dml_sql(models) -%} + {% if models != [] %} + {% set model_values %} + {% for model in models -%} + {% do model.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + ARRAY {{ model.depends_on.nodes }}, {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + '{{ model.original_file_path }}', {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + ARRAY {{ model.tags }}, {# tags #} + '{{ tojson(model.config.meta) | replace("'", "''") }}', {# meta #} + '{{ model.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(model) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 1ccbfe2a..98592540 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -217,3 +217,51 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro trino__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}TIMESTAMP '{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}TIMESTAMP '{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + + {% if model.adapter_response.rows_affected is none %} + null + {% else %} + {{ model.adapter_response.rows_affected }} + {% endif %} + , {# rows_affected #} + + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 32e67383..448466a9 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -109,3 +109,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro trino__get_seeds_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_values %} + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + '{{ tojson(seed.config.meta) | replace("'", "''") }}', {# meta #} + '{{ seed.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(seed) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index 2006b168..5577538d 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -24,7 +24,7 @@ {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }} from values {% for model in snapshots -%} - ( + ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} '{{ run_started_at }}', {# run_started_at #} @@ -33,18 +33,18 @@ {% if config_full_refresh is none %} {% set config_full_refresh = flags.FULL_REFRESH %} {% endif %} - '{{ config_full_refresh }}', {# was_full_refresh #} - - '{{ model.thread_id }}', {# thread_id #} +'{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} - + {{ model.execution_time }}, {# total_node_runtime #} - null, -- rows_affected not available {# Only available in Snowflake #} +null, -- rows_affected not available {# Only available in Snowflake #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} @@ -217,3 +217,51 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro trino__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + + {{ config_full_refresh }}, {# was_full_refresh #} + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}TIMESTAMP '{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}TIMESTAMP '{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + + {% if model.adapter_response.rows_affected is none %} + null + {% else %} + {{ model.adapter_response.rows_affected }} + {% endif %} + , {# rows_affected #} + + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index 0be6759b..bf67ff1a 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -119,3 +119,36 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro trino__get_snapshots_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_values %} + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + ARRAY {{snapshot.depends_on.nodes }}, {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + '{{ tojson(snapshot.config.meta) | replace("'","''") }}', {# meta #} + '{{ snapshot.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(snapshot) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index dbcb49e4..41a4ce07 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -109,3 +109,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro trino__get_sources_dml_sql(sources) -%} + {% if sources != [] %} + {% set source_values %} + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + '{{ source.loaded_at_field | replace("'", "''") }}', {# loaded_at_field #} + '{{ tojson(source.freshness) | replace("'", "''") }}', {# freshness #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(source) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index ea3553ae..21a26aba 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -148,3 +148,50 @@ {{ return("") }} {% endif %} {% endmacro -%} + + +{% macro trino__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + + {{ config_full_refresh }}, {# was_full_refresh #} + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% set compile_started_at = (test.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}TIMESTAMP '{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (test.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}TIMESTAMP '{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ test.execution_time }}, {# total_node_runtime #} + + {% if test.adapter_response.rows_affected is none %} + null + {% else %} + {{ test.adapter_response.rows_affected }} + {% endif %} + , {# rows_affected #} + + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + '{{ test.message | replace("'", "''") }}', {# message #} + '{{ tojson(test.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + + {%- endfor %} + {% endset %} + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index cf75e9ba..3245c295 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -97,3 +97,32 @@ {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro trino__get_tests_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + TIMESTAMP '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + ARRAY {{ test.depends_on.nodes }}, {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path }}', {# test_path #} + ARRAY {{ test.tags }}, {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(test) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 24f1eb77..8d852b87 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -61,3 +61,15 @@ {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} + +{% macro trino__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + values + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} \ No newline at end of file diff --git a/tox.ini b/tox.ini index 542d6e21..bccf7121 100644 --- a/tox.ini +++ b/tox.ini @@ -323,3 +323,51 @@ commands = dbt build --target postgres +#trino integration tests +[testenv:integration_trino] +changedir = integration_test_project +deps = dbt-trino~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target trino + +[testenv:integration_trino_1_3_0] +changedir = integration_test_project +deps = dbt-trino~=1.3.0 +commands = + dbt clean + dbt deps + dbt build --target trino + +[testenv:integration_trino_1_4_0] +changedir = integration_test_project +deps = dbt-trino~=1.4.0 +commands = + dbt clean + dbt deps + dbt build --target trino + +[testenv:integration_trino_1_5_0] +changedir = integration_test_project +deps = dbt-trino~=1.5.0 +commands = + dbt clean + dbt deps + dbt build --target trino + +[testenv:integration_trino_1_6_0] +changedir = integration_test_project +deps = dbt-trino~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target trino + +[testenv:integration_trino_1_7_0] +changedir = integration_test_project +deps = dbt-trino~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target trino \ No newline at end of file