From e03972f07c9cf26ba1e616e870882dcc74215df0 Mon Sep 17 00:00:00 2001 From: Sandy Ryza Date: Fri, 21 Jun 2024 09:11:35 -0700 Subject: [PATCH] add blueprints example --- .../builtin_blueprints/__init__.py | 1 + .../builtin_blueprints/definitions.py | 15 +++++++++++++++ .../pipelines/customer_stats.py | 9 +++++++++ .../pipelines/process_customers.yaml | 8 ++++++++ .../examples/builtin-blueprints/data/.gitkeep | 0 .../examples/builtin-blueprints/pyproject.toml | 6 ++++++ .../examples/builtin-blueprints/setup.cfg | 2 ++ .../examples/builtin-blueprints/setup.py | 13 +++++++++++++ examples/experimental/dagster-blueprints/setup.py | 2 +- 9 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/__init__.py create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/definitions.py create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/customer_stats.py create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/process_customers.yaml create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/data/.gitkeep create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/pyproject.toml create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.cfg create mode 100644 examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.py diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/__init__.py b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/__init__.py new file mode 100644 index 0000000000000..3c25b881e04ff --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/__init__.py @@ -0,0 +1 @@ +from .definitions import defs as defs diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/definitions.py b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/definitions.py new file mode 100644 index 0000000000000..7ddc9af854378 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/definitions.py @@ -0,0 +1,15 @@ +from pathlib import Path +from typing import Sequence + +from dagster import PipesSubprocessClient +from dagster_blueprints import YamlBlueprintsLoader +from dagster_blueprints.shell_command_blueprint import ShellCommandBlueprint + +blueprints_path = Path(__file__).parent / "pipelines" + +loader = YamlBlueprintsLoader( + per_file_blueprint_type=Sequence[ShellCommandBlueprint], path=blueprints_path +) +defs = loader.load_defs( + resources={"pipes_subprocess_client": PipesSubprocessClient(cwd=str(blueprints_path))}, +) diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/customer_stats.py b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/customer_stats.py new file mode 100644 index 0000000000000..20822164f0fee --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/customer_stats.py @@ -0,0 +1,9 @@ +import pandas as pd + +customers = pd.read_csv("../../data/customers.csv") + +num_customers = customers.shape[0] +num_steves = customers[customers["FIRST_NAME"] == "Steve"].shape[0] +customer_stats = pd.DataFrame({"num_customers": [num_customers], "num_steves": [num_steves]}) + +customer_stats.to_csv("../../data/customer_stats.csv", index=False) diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/process_customers.yaml b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/process_customers.yaml new file mode 100644 index 0000000000000..db1844a3a6426 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/builtin_blueprints/pipelines/process_customers.yaml @@ -0,0 +1,8 @@ +- command: "curl -o ../../data/customers.csv https://raw.githubusercontent.com/dagster-io/dagster/master/docs/next/public/assets/customers.csv" + assets: + - key: customers + +- command: "python customer_stats.py" + assets: + - key: customer_stats + deps: [customers] diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/data/.gitkeep b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/data/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/pyproject.toml b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/pyproject.toml new file mode 100644 index 0000000000000..b9f862485aa9f --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "builtin_blueprints" \ No newline at end of file diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.cfg b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.cfg new file mode 100644 index 0000000000000..a03a45324ee91 --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +name = builtin_blueprints diff --git a/examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.py b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.py new file mode 100644 index 0000000000000..aff78f99f34fb --- /dev/null +++ b/examples/experimental/dagster-blueprints/examples/builtin-blueprints/setup.py @@ -0,0 +1,13 @@ +from setuptools import find_packages, setup + +setup( + name="builtin-blueprints", + packages=find_packages(exclude=["builtin-blueprints"]), + install_requires=[ + "dagster", + "dagster-blueprints", + "dagster-webserver", + "pandas", + ], + extras_require={"dev": ["pytest"]}, +) diff --git a/examples/experimental/dagster-blueprints/setup.py b/examples/experimental/dagster-blueprints/setup.py index 1b84630554df5..43b36f6023c28 100644 --- a/examples/experimental/dagster-blueprints/setup.py +++ b/examples/experimental/dagster-blueprints/setup.py @@ -35,7 +35,7 @@ def get_version() -> str: "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - packages=find_packages(exclude=["dagster_blueprints_tests*"]), + packages=find_packages(exclude=["dagster_blueprints_tests*", "examples*"]), install_requires=[ f"dagster{pin}", f"dagster-databricks{pin}",