From 8e2ecb5e1893e9c2717c449917cc3049b79374b2 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Mon, 13 May 2024 14:42:34 -0400 Subject: [PATCH 1/3] Add dbt quickstart example for already existing Dagster project --- docs/content/integrations/dbt/quickstart.mdx | 61 +++++++++++++++++++ .../integrations/dbt/quickstart.py | 48 +++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/docs/content/integrations/dbt/quickstart.mdx b/docs/content/integrations/dbt/quickstart.mdx index 55d8103865b0e..3fdcf8205f978 100644 --- a/docs/content/integrations/dbt/quickstart.mdx +++ b/docs/content/integrations/dbt/quickstart.mdx @@ -63,6 +63,67 @@ dagster-dbt project scaffold --project-name my_dagster_project --dbt-project-dir This command creates a new directory called `my_dagster_project/` inside the current directory. The new `my_dagster_project/` directory will contain a set of files that define a Dagster project to load the dbt project provided in `./my_dbt_project`. + + + +### Option 2: Use an existing Dagster project + +You can use an existing Dagster project to run your dbt project. To do so, you'll need to add some objects using the [dagster-dbt library](/\_apidocs/libraries/dagster-dbt) and update your `Definitions` object. This example assumes that your existing Dagster project includes both `assets.py` and `definitions.py` files. + +1. Change directories to the Dagster project directory: + + ```shell + cd my-dagster-project/ + ``` + +2. Create a Python file named `project.py` and add the following code. The DbtProject object is a representation of your dbt project that assist with managing manifest.json preparation. + + ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_project_example endbefore=end_dbt_project_example + from pathlib import Path + + from dagster_dbt import DbtProject + + RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my-dbt-project" + + my_project = DbtProject( + project_dir=Path(__file__) + .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) + .resolve(), + ) + ``` + +3. In your `assets.py` file, add the following code. Using the `dbt_assets` decorator allows Dagster to create a definition for how to compute a set of dbt resources, described by a manifest.json. + + ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_assets_example endbefore=end_dbt_assets_example + from dagster import AssetExecutionContext + from dagster_dbt import DbtCliResource, dbt_assets + + from .project import my_project + + @dbt_assets(manifest=my_project.manifest_path) + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): + yield from dbt.cli(["build"], context=context).stream() + ``` + +4. In your `definitions.py` file, update your Definitions object to include the newly created objects. + + ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_definitions_example endbefore=end_dbt_definitions_example + from dagster import Definitions + from dagster_dbt import DbtCliResource + + from .assets import my_dbt_assets + from .project import my_project + + defs = Definitions( + assets=[my_dbt_assets], + resources={ + "dbt": DbtCliResource(project_dir=my_project), + }, + ) + ``` + +With these changes, your existing Dagster project is ready to run your dbt project. + diff --git a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py index 35c17175f8d84..e873ffd8aeee1 100644 --- a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py +++ b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false # ruff: isort: skip_file # start_example @@ -40,3 +41,50 @@ def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): }, ) # end_example + + +def dbt_project_example(): + # start_dbt_project_example + from pathlib import Path + + from dagster_dbt import DbtProject + + RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my-dbt-project" + + my_project = DbtProject( + project_dir=Path(__file__) + .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) + .resolve(), + ) + # end_dbt_project_example + + +def dbt_assets_example(): + # start_dbt_assets_example + from dagster import AssetExecutionContext + from dagster_dbt import DbtCliResource, dbt_assets + + from .project import my_project + + @dbt_assets(manifest=my_project.manifest_path) + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): + yield from dbt.cli(["build"], context=context).stream() + + # end_dbt_assets_example + + +def dbt_definitions_example(): + # start_dbt_definitions_example + from dagster import Definitions + from dagster_dbt import DbtCliResource + + from .assets import my_dbt_assets + from .project import my_project + + defs = Definitions( + assets=[my_dbt_assets], + resources={ + "dbt": DbtCliResource(project_dir=my_project), + }, + ) + # end_dbt_definitions_example From 987964260d40e9ade59f8c8fe35533b42386e4d9 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Tue, 14 May 2024 12:40:42 -0400 Subject: [PATCH 2/3] Update quickstart post-review --- docs/content/integrations/dbt/quickstart.mdx | 31 ++++++++++++++----- .../integrations/dbt/quickstart.py | 11 +++++-- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/content/integrations/dbt/quickstart.mdx b/docs/content/integrations/dbt/quickstart.mdx index 3fdcf8205f978..12a77754a7aa6 100644 --- a/docs/content/integrations/dbt/quickstart.mdx +++ b/docs/content/integrations/dbt/quickstart.mdx @@ -66,24 +66,34 @@ This command creates a new directory called `my_dagster_project/` inside the cur -### Option 2: Use an existing Dagster project +### Option 2: Load your dbt project in an existing Dagster project -You can use an existing Dagster project to run your dbt project. To do so, you'll need to add some objects using the [dagster-dbt library](/\_apidocs/libraries/dagster-dbt) and update your `Definitions` object. This example assumes that your existing Dagster project includes both `assets.py` and `definitions.py` files. +You can use an existing Dagster project to run your dbt project. To do so, you'll need to add some objects using the [dagster-dbt library](/\_apidocs/libraries/dagster-dbt) and update your `Definitions` object. This example assumes that your existing Dagster project includes both `assets.py` and `definitions.py` files, among other required files. + +```shell +my_dagster_project +├── __init__.py +├── assets.py +├── definitions.py +├── pyproject.toml +├── setup.cfg +└── setup.py +``` 1. Change directories to the Dagster project directory: ```shell - cd my-dagster-project/ + cd my_dagster_project/ ``` -2. Create a Python file named `project.py` and add the following code. The DbtProject object is a representation of your dbt project that assist with managing manifest.json preparation. +2. Create a Python file named `project.py` and add the following code. The DbtProject object is a representation of your dbt project that assist with managing `manifest.json` preparation. ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_project_example endbefore=end_dbt_project_example from pathlib import Path from dagster_dbt import DbtProject - RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my-dbt-project" + RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my_dbt_project" my_project = DbtProject( project_dir=Path(__file__) @@ -92,7 +102,7 @@ You can use an existing Dagster project to run your dbt project. To do so, you'l ) ``` -3. In your `assets.py` file, add the following code. Using the `dbt_assets` decorator allows Dagster to create a definition for how to compute a set of dbt resources, described by a manifest.json. +3. In your `assets.py` file, add the following code. Using the `dbt_assets` decorator allows Dagster to create a definition for how to compute a set of dbt resources, described by a `manifest.json`. ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_assets_example endbefore=end_dbt_assets_example from dagster import AssetExecutionContext @@ -115,8 +125,15 @@ You can use an existing Dagster project to run your dbt project. To do so, you'l from .project import my_project defs = Definitions( - assets=[my_dbt_assets], + assets=[ + # your other assets here, + my_dbt_assets + ], + jobs=[ + # your jobs here + ], resources={ + # your other resources here, "dbt": DbtCliResource(project_dir=my_project), }, ) diff --git a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py index e873ffd8aeee1..39aa92766ab15 100644 --- a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py +++ b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py @@ -49,7 +49,7 @@ def dbt_project_example(): from dagster_dbt import DbtProject - RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my-dbt-project" + RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my_dbt_project" my_project = DbtProject( project_dir=Path(__file__) @@ -82,8 +82,15 @@ def dbt_definitions_example(): from .project import my_project defs = Definitions( - assets=[my_dbt_assets], + assets=[ + # your other assets here, + my_dbt_assets + ], + jobs=[ + # your jobs here + ], resources={ + # your other resources here, "dbt": DbtCliResource(project_dir=my_project), }, ) From d86c381a45412650c8c9fbfcd0d9313f6374f8a5 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Tue, 14 May 2024 14:30:21 -0400 Subject: [PATCH 3/3] Update code snippets post-review --- docs/content/integrations/dbt/quickstart.mdx | 60 +++++----- .../integrations/dbt/quickstart.py | 107 +++++++++--------- 2 files changed, 85 insertions(+), 82 deletions(-) diff --git a/docs/content/integrations/dbt/quickstart.mdx b/docs/content/integrations/dbt/quickstart.mdx index 12a77754a7aa6..bc67885f1ba06 100644 --- a/docs/content/integrations/dbt/quickstart.mdx +++ b/docs/content/integrations/dbt/quickstart.mdx @@ -91,52 +91,52 @@ my_dagster_project ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_project_example endbefore=end_dbt_project_example from pathlib import Path - from dagster_dbt import DbtProject + from dagster_dbt import DbtProject - RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my_dbt_project" + RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my_dbt_project" - my_project = DbtProject( - project_dir=Path(__file__) - .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) - .resolve(), - ) + my_project = DbtProject( + project_dir=Path(__file__) + .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) + .resolve(), + ) ``` 3. In your `assets.py` file, add the following code. Using the `dbt_assets` decorator allows Dagster to create a definition for how to compute a set of dbt resources, described by a `manifest.json`. ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_assets_example endbefore=end_dbt_assets_example from dagster import AssetExecutionContext - from dagster_dbt import DbtCliResource, dbt_assets + from dagster_dbt import DbtCliResource, dbt_assets - from .project import my_project + from .project import my_project - @dbt_assets(manifest=my_project.manifest_path) - def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): - yield from dbt.cli(["build"], context=context).stream() + @dbt_assets(manifest=my_project.manifest_path) + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): + yield from dbt.cli(["build"], context=context).stream() ``` 4. In your `definitions.py` file, update your Definitions object to include the newly created objects. ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_definitions_example endbefore=end_dbt_definitions_example from dagster import Definitions - from dagster_dbt import DbtCliResource - - from .assets import my_dbt_assets - from .project import my_project - - defs = Definitions( - assets=[ - # your other assets here, - my_dbt_assets - ], - jobs=[ - # your jobs here - ], - resources={ - # your other resources here, - "dbt": DbtCliResource(project_dir=my_project), - }, - ) + from dagster_dbt import DbtCliResource + + from .assets import my_dbt_assets + from .project import my_project + + defs = Definitions( + ..., + assets=[ + ..., + # Add the dbt assets alongside your other asset + my_dbt_assets + ], + resources={ + ..., + # Add the dbt resource alongside your other resources + "dbt": DbtCliResource(project_dir=my_project), + }, + ) ``` With these changes, your existing Dagster project is ready to run your dbt project. diff --git a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py index 39aa92766ab15..cf474c4c83d70 100644 --- a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py +++ b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py @@ -43,55 +43,58 @@ def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): # end_example -def dbt_project_example(): - # start_dbt_project_example - from pathlib import Path - - from dagster_dbt import DbtProject - - RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my_dbt_project" - - my_project = DbtProject( - project_dir=Path(__file__) - .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) - .resolve(), - ) - # end_dbt_project_example - - -def dbt_assets_example(): - # start_dbt_assets_example - from dagster import AssetExecutionContext - from dagster_dbt import DbtCliResource, dbt_assets - - from .project import my_project - - @dbt_assets(manifest=my_project.manifest_path) - def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): - yield from dbt.cli(["build"], context=context).stream() - - # end_dbt_assets_example - - -def dbt_definitions_example(): - # start_dbt_definitions_example - from dagster import Definitions - from dagster_dbt import DbtCliResource - - from .assets import my_dbt_assets - from .project import my_project - - defs = Definitions( - assets=[ - # your other assets here, - my_dbt_assets - ], - jobs=[ - # your jobs here - ], - resources={ - # your other resources here, - "dbt": DbtCliResource(project_dir=my_project), - }, - ) - # end_dbt_definitions_example +# start_dbt_project_example +from pathlib import Path + +from dagster_dbt import DbtProject + +RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my_dbt_project" + +my_project = DbtProject( + project_dir=Path(__file__) + .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) + .resolve(), +) +# end_dbt_project_example + + +# Using a string to avoid ruff adding a second blank line before the dbt_assets. +""" +# start_dbt_assets_example +from dagster import AssetExecutionContext +from dagster_dbt import DbtCliResource, dbt_assets + +from .project import my_project + +@dbt_assets(manifest=my_project.manifest_path) +def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): + yield from dbt.cli(["build"], context=context).stream() +# end_dbt_assets_example +""" + + +# Using a string to avoid compilation error caused by the `...` placeholders +"""# start_dbt_definitions_example + +from dagster import Definitions +from dagster_dbt import DbtCliResource + +from .assets import my_dbt_assets +from .project import my_project + +defs = Definitions( + ..., + assets=[ + ..., + # Add the dbt assets alongside your other asset + my_dbt_assets + ], + resources={ + ..., + # Add the dbt resource alongside your other resources + "dbt": DbtCliResource(project_dir=my_project), + }, +) + +# end_dbt_definitions_example +"""