From 7d0cf6ddb70139155f5ab9ddca59a4adca055183 Mon Sep 17 00:00:00 2001 From: David Griffiths Date: Wed, 1 May 2024 01:13:03 +1000 Subject: [PATCH] fix(dbt): support projects with saved queries (#21441) ## Summary & Motivation When i upgrade from dagster 1.6.11 to 1.7.3 I am no longer able to use [dbt's saved queries](https://docs.getdbt.com/docs/build/saved-queries) via the [dagster-dbt integration](https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt). I am now getting a `key error` before any definitions can be loaded by dagster from the dagster-dbt assets example: image For reference `saved_queries` are located in the [dbt's manifest.json schema](https://schemas.getdbt.com/dbt/manifest/v11/index.html#saved_queries) ## The problem Manifest creation in master (dagster 1.7.3): https://github.com/dagster-io/dagster/blob/master/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py#L255 has changed how the manifest is parsed and no longer picks up saved_queries. Manifest creation in master (dagster 1.6.11): https://github.com/dagster-io/dagster/blob/release-1.6.11/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py#L243 is the version that currently works for me. It simply gets all keys from dbt's manifest.json with the code`Manifest.from_dict(manifest_json)` I have added the `saved_queries` key to this branch to fix this up ## How I Tested These Changes Have tested this locally (editable install) and it allows me to load all definitions including dbt's saved queries. --- .../dagster-dbt/dagster_dbt/asset_utils.py | 1 + .../dagster-dbt/dagster_dbt/utils.py | 13 ++++++++++- .../models/schema.yml | 16 ------------- .../models/semantic_model.yml | 23 +++++++++++++++++++ 4 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/semantic_model.yml diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/asset_utils.py b/python_modules/libraries/dagster-dbt/dagster_dbt/asset_utils.py index 0e87e21c22001..fbb0a4e5d4cf3 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/asset_utils.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/asset_utils.py @@ -641,6 +641,7 @@ def is_non_asset_node(dbt_resource_props: Mapping[str, Any]): [ resource_type == "metric", resource_type == "semantic_model", + resource_type == "saved_query", resource_type == "model" and dbt_resource_props.get("config", {}).get("materialized") == "ephemeral", ] diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py b/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py index a5708717e1f50..ec41123a55844 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py @@ -273,12 +273,22 @@ def __getattr__(self, item): { "semantic_models": { unique_id: _DictShim(info) - for unique_id, info in manifest_json.get("semantic_models", {}).items() + for unique_id, info in manifest_json["semantic_models"].items() } } if manifest_json.get("semantic_models") else {} ), + **( + { + "saved_queries": { + unique_id: _DictShim(info) + for unique_id, info in manifest_json["saved_queries"].items() + }, + } + if manifest_json.get("saved_queries") + else {} + ), ) child_map = manifest_json["child_map"] @@ -313,6 +323,7 @@ def get_dbt_resource_props_by_dbt_unique_id_from_manifest( **manifest["exposures"], **manifest["metrics"], **manifest.get("semantic_models", {}), + **manifest.get("saved_queries", {}), } diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/schema.yml b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/schema.yml index c98f956082064..05c5be6930f7f 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/schema.yml +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/schema.yml @@ -81,19 +81,3 @@ models: description: Amount of the order (AUD) paid for by gift card tests: - not_null - -semantic_models: - - name: customers - description: Customer grain mart. - model: ref('customers') - entities: - - name: customer - expr: customer_id - type: primary - measures: - - name: total_order_amount - description: Total count of orders per customer. - agg: sum - - name: total_order_amount - agg: sum - description: Gross customer lifetime spend inclusive of taxes. diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/semantic_model.yml b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/semantic_model.yml new file mode 100644 index 0000000000000..1b8adbc7601c6 --- /dev/null +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/dbt_projects/test_dagster_dbt_semantic_models/models/semantic_model.yml @@ -0,0 +1,23 @@ +version: 2 + +semantic_models: + - name: customers + description: Customer grain mart. + model: ref('customers') + entities: + - name: customer + expr: customer_id + type: primary + measures: + - name: total_order_amount + description: Total count of orders per customer. + agg: sum + - name: total_order_amount + agg: sum + description: Gross customer lifetime spend inclusive of taxes. + +saved_queries: + - name: customers_query + description: Customer query + label: Customer query + query_params: {}