From 9ffbc7f814433aeeffd449bf7a97d745e7c8ce32 Mon Sep 17 00:00:00 2001 From: Sandy Ryza Date: Fri, 21 Jun 2024 09:30:58 -0700 Subject: [PATCH] add example with custom blueprints --- .../custom_blueprints/__init__.py | 4 ++ .../curl_assets/cereals.yaml | 3 ++ .../curl_assets/customers.yaml | 3 ++ .../custom_blueprints/definitions.py | 47 +++++++++++++++++++ .../examples/custom-blueprints/pyproject.toml | 6 +++ .../examples/custom-blueprints/setup.cfg | 2 + .../examples/custom-blueprints/setup.py | 12 +++++ 7 files changed, 77 insertions(+) create mode 100644 examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/__init__.py create mode 100644 examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/cereals.yaml create mode 100644 examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/customers.yaml create mode 100644 examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/definitions.py create mode 100644 examples/experimental/dagster-blueprints/examples/custom-blueprints/pyproject.toml create mode 100644 examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.cfg create mode 100644 examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.py diff --git a/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/__init__.py b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/__init__.py new file mode 100644 index 0000000000000..a328dc49a3134 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/__init__.py @@ -0,0 +1,4 @@ +from .definitions import ( + defs as defs, + loader as loader, +) diff --git a/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/cereals.yaml b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/cereals.yaml new file mode 100644 index 0000000000000..3e6ef6be58626 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/cereals.yaml @@ -0,0 +1,3 @@ +type: curl_asset +table_name: cereals +csv_url: https://raw.githubusercontent.com/dagster-io/dagster/master/docs/next/public/assets/cereal.csv diff --git a/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/customers.yaml b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/customers.yaml new file mode 100644 index 0000000000000..531378377a6e6 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/curl_assets/customers.yaml @@ -0,0 +1,3 @@ +type: curl_asset +table_name: customers +csv_url: https://raw.githubusercontent.com/dagster-io/dagster/master/docs/next/public/assets/customers.csv diff --git a/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/definitions.py b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/definitions.py new file mode 100644 index 0000000000000..e57f2a4ab0cd8 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/custom-blueprints/custom_blueprints/definitions.py @@ -0,0 +1,47 @@ +import tempfile +from pathlib import Path +from subprocess import Popen +from typing import Literal + +from dagster import asset +from dagster_blueprints import YamlBlueprintsLoader +from dagster_blueprints.blueprint import Blueprint, BlueprintDefinitions + + +def local_csv_to_snowflake(table_name: str, csv_path: Path) -> None: + """Left as an exercise to the reader.""" + + +def curl_csv_to_snowflake(table_name: str, csv_url: str) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + path = Path(tmp_dir) / "file.csv" + command = ["curl", "-o", str(path), csv_url] + process = Popen(command) + process.wait() + + if process.returncode != 0: + raise ValueError(f"External execution process failed with code {process.returncode}") + + local_csv_to_snowflake(table_name, path) + + +class CurlCsvSnowflakeAssetBlueprint(Blueprint): + """Blueprint for an asset in Snowflake, populated from a CSV file on the internet.""" + + type: Literal["curl_asset"] + csv_url: str + table_name: str + + def build_defs(self) -> BlueprintDefinitions: + @asset(key=self.table_name) + def _asset(): + curl_csv_to_snowflake(table_name=self.table_name, csv_url=self.csv_url) + + return BlueprintDefinitions(assets=[_asset]) + + +loader = YamlBlueprintsLoader( + per_file_blueprint_type=CurlCsvSnowflakeAssetBlueprint, + path=Path(__file__).parent / "curl_assets", +) +defs = loader.load_defs() diff --git a/examples/experimental/dagster-blueprints/examples/custom-blueprints/pyproject.toml b/examples/experimental/dagster-blueprints/examples/custom-blueprints/pyproject.toml new file mode 100644 index 0000000000000..939146f16f1e8 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/custom-blueprints/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "custom_blueprints" \ No newline at end of file diff --git a/examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.cfg b/examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.cfg new file mode 100644 index 0000000000000..add6ee9d79cfb --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +name = custom_blueprints diff --git a/examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.py b/examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.py new file mode 100644 index 0000000000000..ff72aa4465ebb --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/custom-blueprints/setup.py @@ -0,0 +1,12 @@ +from setuptools import find_packages, setup + +setup( + name="custom-blueprints", + packages=find_packages(exclude=["custom-blueprints"]), + install_requires=[ + "dagster", + "dagster-blueprints", + "dagster-webserver", + ], + extras_require={"dev": ["pytest"]}, +)