Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Clickhouse feature; getting columns from source; fix field name and table path #137

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Clickhouse feature; getting columns from source; fix field name and table path #137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5ab9fec
Fix missing column catching; fix join algorithm due to "join on false…
gladkikhtutu Mar 16, 2023
e1cd2a9
fix path to table
gladkikhtutu Mar 16, 2023
f57c78e
fix count field collision
gladkikhtutu Mar 16, 2023
71a5ad0
Merge branch 'EqualExperts:master' into master
gladkikhtutu Apr 24, 2023
5057bc5
refactor to create PR
gladkikhtutu Apr 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion macros/mock_builders.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@
{% set model_columns = dbt_unit_testing.get_from_cache("COLUMNS", model_node.name) %}
{% if not model_columns %}
{% set model_sql = dbt_unit_testing.build_node_sql(model_node, complete=true, use_database_models=options.use_database_models) %}
{% set model_columns = dbt_unit_testing.extract_columns_list(model_sql) %}
{% set sql_model_columns = dbt_unit_testing.extract_columns_list(model_sql) %}
{% set source_model_columns = model_node.columns.values() | map(attribute='name') | list %}
{% set model_columns = (sql_model_columns + source_model_columns) | unique | list %}
{{ dbt_unit_testing.cache("COLUMNS", model_node.name, model_columns)}}
{% else %}
{{ dbt_unit_testing.verbose("CACHE HIT for " ~ model_node.name ~ " COLUMNS") }}
Expand Down
11 changes: 10 additions & 1 deletion macros/sql_builders.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
{%- endif -%}
{{ "\n" }}
select * from ({{ dbt_unit_testing.render_node(model_node) }} {{ "\n" }} ) as t
{% if adapter.type() == 'clickhouse' %}
SETTINGS join_algorithm='hash'
{% endif %}
{%- endset -%}

{% do return(model_complete_sql) %}
Expand Down Expand Up @@ -85,7 +88,13 @@
{% set name = node.name %}
{%- endif %}

select * from {{ dbt_unit_testing.quote_identifier(node.database) ~ '.' ~ dbt_unit_testing.quote_identifier(node.schema) ~ '.' ~ dbt_unit_testing.quote_identifier(name) }} where false
{%- if node.database is none %}
{% set table_path = dbt_unit_testing.quote_identifier(node.schema) ~ '.' ~ dbt_unit_testing.quote_identifier(name) %}
{%- else %}
{% set table_path = dbt_unit_testing.quote_identifier(node.database) ~ '.' ~ dbt_unit_testing.quote_identifier(node.schema) ~ '.' ~ dbt_unit_testing.quote_identifier(name) %}
{%- endif %}

select * from {{ table_path }} where false
{%- else -%}
{% if complete %}
{{ dbt_unit_testing.build_model_complete_sql(node) }}
Expand Down
26 changes: 13 additions & 13 deletions macros/tests.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@

{% if execute %}
{% set test_configuration = {
"model_name": model_name,
"description": test_description,
"options": dbt_unit_testing.merge_configs([options])}
"model_name": model_name,
"description": test_description,
"options": dbt_unit_testing.merge_configs([options])}
%}
{% set mocks_and_expectations_json_str = caller() %}

{{ dbt_unit_testing.verbose("CONFIG: " ~ test_configuration) }}

{% do test_configuration.update (dbt_unit_testing.build_mocks_and_expectations(test_configuration, mocks_and_expectations_json_str)) %}
{% set test_report = dbt_unit_testing.build_test_report(test_configuration) %}

{% if not test_report.succeeded %}
{{ dbt_unit_testing.show_test_report(test_configuration, test_report) }}
{% endif %}

select 1 as a from (select 1) as t where {{ not test_report.succeeded }}
{% endif %}
{% endmacro %}
Expand Down Expand Up @@ -69,11 +69,11 @@
{% set columns = dbt_unit_testing.quote_and_join_columns(dbt_unit_testing.extract_columns_list(expectations.input_values)) %}

{%- set actual_query -%}
select count(1) as count, {{columns}} from ( {{ model_complete_sql }} ) as s group by {{ columns }}
select count(1) as count_stat, {{columns}} from ( {{ model_complete_sql }} ) as s group by {{ columns }}
{% endset %}

{%- set expectations_query -%}
select count(1) as count, {{columns}} from ({{ expectations.input_values }}) as s group by {{ columns }}
select count(1) as count_stat, {{columns}} from ({{ expectations.input_values }}) as s group by {{ columns }}
{% endset %}

{%- set test_query -%}
Expand All @@ -85,15 +85,15 @@
),

extra_entries as (
select '+' as diff, count, {{columns}} from actual
select '+' as diff, count_stat, {{columns}} from actual
{{ except() }}
select '+' as diff, count, {{columns}} from expectations),
select '+' as diff, count_stat, {{columns}} from expectations),

missing_entries as (
select '-' as diff, count, {{columns}} from expectations
select '-' as diff, count_stat, {{columns}} from expectations
{{ except() }}
select '-' as diff, count, {{columns}} from actual)
select '-' as diff, count_stat, {{columns}} from actual)

select * from extra_entries
UNION ALL
select * from missing_entries
Expand Down Expand Up @@ -136,7 +136,7 @@
{% set test_query = test_queries.test_query %}

{%- set count_query -%}
select * FROM
select * FROM
(select count(1) as expectation_count from (
{{ expectations_query }}
) as exp) as exp_count,
Expand Down