-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
revert defs changes to examples (#23322)
## Summary Reverts changes to our examples which move the defs import path as they are not compatible with the cloud NUX.
- Loading branch information
Showing
73 changed files
with
315 additions
and
318 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
import os | ||
|
||
from dagster import ( | ||
Definitions, | ||
FilesystemIOManager, | ||
ScheduleDefinition, | ||
define_asset_job, | ||
load_assets_from_package_module, | ||
) | ||
from dagster._core.definitions.asset_check_factories.freshness_checks.sensor import ( | ||
build_sensor_for_freshness_checks, | ||
) | ||
from dagster_duckdb_pandas import DuckDBPandasIOManager | ||
|
||
from .assets import forecasting, raw_data | ||
from .assets.dbt import DBT_PROJECT_DIR, dbt_project_assets, dbt_resource | ||
|
||
raw_data_assets = load_assets_from_package_module( | ||
raw_data, | ||
group_name="raw_data", | ||
# all of these assets live in the duckdb database, under the schema raw_data | ||
key_prefix=["duckdb", "raw_data"], | ||
) | ||
|
||
forecasting_assets = load_assets_from_package_module( | ||
forecasting, | ||
group_name="forecasting", | ||
) | ||
all_assets_checks = [*forecasting.forecasting_freshness_checks] | ||
|
||
# The freshness check sensor will run our freshness checks even if the underlying asset fails to run, for whatever reason. | ||
freshness_check_sensor = build_sensor_for_freshness_checks(freshness_checks=all_assets_checks) | ||
|
||
# define jobs as selections over the larger graph | ||
everything_job = define_asset_job("everything_everywhere_job", selection="*") | ||
forecast_job = define_asset_job("refresh_forecast_model_job", selection="*order_forecast_model") | ||
|
||
resources = { | ||
# this io_manager allows us to load dbt models as pandas dataframes | ||
"io_manager": DuckDBPandasIOManager(database=os.path.join(DBT_PROJECT_DIR, "example.duckdb")), | ||
# this io_manager is responsible for storing/loading our pickled machine learning model | ||
"model_io_manager": FilesystemIOManager(), | ||
# this resource is used to execute dbt cli commands | ||
"dbt": dbt_resource, | ||
} | ||
|
||
defs = Definitions( | ||
assets=[dbt_project_assets, *raw_data_assets, *forecasting_assets], | ||
resources=resources, | ||
asset_checks=all_assets_checks, | ||
schedules=[ | ||
ScheduleDefinition(job=everything_job, cron_schedule="@weekly"), | ||
ScheduleDefinition(job=forecast_job, cron_schedule="@daily"), | ||
], | ||
sensors=[freshness_check_sensor], | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
56 changes: 0 additions & 56 deletions
56
examples/assets_dbt_python/assets_dbt_python/definitions.py
This file was deleted.
Oops, something went wrong.
File renamed without changes.
2 changes: 1 addition & 1 deletion
2
examples/assets_dbt_python/assets_dbt_python_tests/test_defs.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
from assets_dbt_python.definitions import defs | ||
from assets_dbt_python import defs | ||
|
||
|
||
def test_def_can_load(): | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
from dagster import Definitions, load_assets_from_modules | ||
from dagster_duckdb import build_duckdb_io_manager | ||
from dagster_duckdb_pandas import DuckDBPandasTypeHandler | ||
|
||
from . import assets | ||
from .release_sensor import release_sensor | ||
|
||
duckdb_io_manager = build_duckdb_io_manager([DuckDBPandasTypeHandler()]) | ||
|
||
defs = Definitions( | ||
assets=load_assets_from_modules([assets]), | ||
sensors=[release_sensor], | ||
resources={"warehouse": duckdb_io_manager.configured({"database": "releases.duckdb"})}, | ||
) |
14 changes: 0 additions & 14 deletions
14
examples/assets_dynamic_partitions/assets_dynamic_partitions/definitions.py
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
from dagster import ( | ||
Definitions, | ||
ScheduleDefinition, | ||
define_asset_job, | ||
load_assets_from_package_module, | ||
) | ||
from dagster._core.definitions.asset_check_factories.freshness_checks.sensor import ( | ||
build_sensor_for_freshness_checks, | ||
) | ||
from dagster_airbyte import AirbyteResource | ||
|
||
from . import assets | ||
from .assets.forecasting import freshness_checks | ||
from .db_io_manager import DbIOManager | ||
from .utils.constants import AIRBYTE_CONFIG, POSTGRES_CONFIG, dbt_resource | ||
|
||
# The freshness check sensor will run our freshness checks even if the underlying asset fails to run, for whatever reason. | ||
freshness_check_sensor = build_sensor_for_freshness_checks(freshness_checks=freshness_checks) | ||
|
||
defs = Definitions( | ||
assets=load_assets_from_package_module(assets), | ||
resources={ | ||
"airbyte": AirbyteResource(**AIRBYTE_CONFIG), | ||
"dbt": dbt_resource, | ||
"db_io_manager": DbIOManager(**POSTGRES_CONFIG), | ||
}, | ||
asset_checks=freshness_checks, | ||
schedules=[ | ||
# update all assets once a day | ||
ScheduleDefinition( | ||
job=define_asset_job("all_assets", selection="*"), cron_schedule="@daily" | ||
), | ||
], | ||
sensors=[freshness_check_sensor], | ||
) |
35 changes: 0 additions & 35 deletions
35
examples/assets_modern_data_stack/assets_modern_data_stack/definitions.py
This file was deleted.
Oops, something went wrong.
2 changes: 1 addition & 1 deletion
2
examples/assets_modern_data_stack/assets_modern_data_stack_tests/test_defs.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
examples/assets_pandas_pyspark/assets_pandas_pyspark/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,42 @@ | ||
def get_weather_defs(): | ||
# gather_assets_start | ||
|
||
# __init__.py | ||
from dagster import Definitions, load_assets_from_modules | ||
|
||
from .assets import table_assets | ||
from .local_filesystem_io_manager import LocalFileSystemIOManager | ||
|
||
defs = Definitions( | ||
# imports the module called "assets" from the package containing the current module | ||
# the "assets" module contains the asset definitions | ||
assets=load_assets_from_modules([table_assets]), | ||
resources={ | ||
"io_manager": LocalFileSystemIOManager(), | ||
}, | ||
) | ||
# gather_assets_end | ||
|
||
return defs | ||
|
||
|
||
def get_spark_weather_defs(): | ||
# gather_spark_assets_start | ||
|
||
# __init__.py | ||
|
||
from dagster import Definitions, load_assets_from_modules | ||
|
||
from .assets import spark_asset, table_assets | ||
from .local_spark_filesystem_io_manager import LocalFileSystemIOManager | ||
|
||
defs = Definitions( | ||
assets=load_assets_from_modules([table_assets, spark_asset]), | ||
resources={"io_manager": LocalFileSystemIOManager()}, | ||
) | ||
# gather_spark_assets_end | ||
|
||
return defs | ||
|
||
|
||
defs = get_spark_weather_defs() |
42 changes: 0 additions & 42 deletions
42
examples/assets_pandas_pyspark/assets_pandas_pyspark/definitions.py
This file was deleted.
Oops, something went wrong.
2 changes: 1 addition & 1 deletion
2
examples/assets_pandas_pyspark/assets_pandas_pyspark_tests/test_defs.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
from dagster import Definitions, load_assets_from_package_module | ||
|
||
from . import ( | ||
assets, | ||
lib as lib, | ||
) | ||
from .resources.csv_io_manager import LocalCsvIOManager | ||
|
||
defs = Definitions( | ||
assets=load_assets_from_package_module(assets), resources={"io_manager": LocalCsvIOManager()} | ||
) |
Oops, something went wrong.
2a86a20
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Deploy preview for dagster-docs ready!
✅ Preview
https://dagster-docs-i57wlbo9k-elementl.vercel.app
https://master.dagster.dagster-docs.io
Built with commit 2a86a20.
This pull request is being automatically deployed with vercel-action