From 58b65180c3c59f354138b480854ba3f32b22529e Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Mon, 13 May 2024 14:42:34 -0400 Subject: [PATCH] Add dbt quickstart example for already existing Dagster project --- docs/content/integrations/dbt/quickstart.mdx | 61 +++++++++++++++++++ .../integrations/dbt/quickstart.py | 48 +++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/docs/content/integrations/dbt/quickstart.mdx b/docs/content/integrations/dbt/quickstart.mdx index e5873df881d43..ed3ad87bbcf2d 100644 --- a/docs/content/integrations/dbt/quickstart.mdx +++ b/docs/content/integrations/dbt/quickstart.mdx @@ -58,6 +58,67 @@ dagster-dbt project scaffold --project-name my-dagster-project --dbt-project-dir This creates a directory called `my-dagster-project/` inside the current directory. The `my-dagster-project/` directory contains a set of files that define a Dagster project. + + + +### Option 2: Use an existing Dagster project + +You can use an existing Dagster project to run your dbt project. To do so, you'll need to add some objects using the [dagster-dbt library](/\_apidocs/libraries/dagster-dbt) and update your `Definitions` object. This example assumes that your existing Dagster project includes both `assets.py` and `definitions.py` files. + +1. Change directories to the Dagster project directory: + + ```shell + cd my-dagster-project/ + ``` + +2. Create a Python file named `project.py` and add the following code. The DbtProject object is a representation of your dbt project that assist with managing manifest.json preparation. + + ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_project_example endbefore=end_dbt_project_example + from pathlib import Path + + from dagster_dbt import DbtProject + + RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my-dbt-project" + + my_project = DbtProject( + project_dir=Path(__file__) + .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) + .resolve(), + ) + ``` + +3. In your `assets.py` file, add the following code. Using the `dbt_assets` decorator allows Dagster to create a definition for how to compute a set of dbt resources, described by a manifest.json. + + ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_assets_example endbefore=end_dbt_assets_example + from dagster import AssetExecutionContext + from dagster_dbt import DbtCliResource, dbt_assets + + from .project import my_project + + @dbt_assets(manifest=my_project.manifest_path) + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): + yield from dbt.cli(["build"], context=context).stream() + ``` + +4. In your `definitions.py` file, update your Definitions object to include the newly created objects. + + ```python file=/integrations/dbt/quickstart.py startafter=start_dbt_definitions_example endbefore=end_dbt_definitions_example + from dagster import Definitions + from dagster_dbt import DbtCliResource + + from .assets import my_dbt_assets + from .project import my_project + + defs = Definitions( + assets=[my_dbt_assets], + resources={ + "dbt": DbtCliResource(project_dir=my_project), + }, + ) + ``` + +With these changes, your existing Dagster project is ready to run your dbt project. + diff --git a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py index 0fb273e9436f9..74d0b13e77f3a 100644 --- a/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py +++ b/examples/docs_snippets/docs_snippets/integrations/dbt/quickstart.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false # ruff: isort: skip_file # start_example @@ -40,3 +41,50 @@ def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): }, ) # end_example + + +def dbt_project_example(): + # start_dbt_project_example + from pathlib import Path + + from dagster_dbt import DbtProject + + RELATIVE_PATH_TO_MY_DBT_PROJECT = "./my-dbt-project" + + my_project = DbtProject( + project_dir=Path(__file__) + .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT) + .resolve(), + ) + # end_dbt_project_example + + +def dbt_assets_example(): + # start_dbt_assets_example + from dagster import AssetExecutionContext + from dagster_dbt import DbtCliResource, dbt_assets + + from .project import my_project + + @dbt_assets(manifest=my_project.manifest_path) + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): + yield from dbt.cli(["build"], context=context).stream() + + # end_dbt_assets_example + + +def dbt_definitions_example(): + # start_dbt_definitions_example + from dagster import Definitions + from dagster_dbt import DbtCliResource + + from .assets import my_dbt_assets + from .project import my_project + + defs = Definitions( + assets=[my_dbt_assets], + resources={ + "dbt": DbtCliResource(project_dir=my_project), + }, + ) + # end_dbt_definitions_example