diff --git a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config/replication.yaml b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config/replication.yaml index 81f2cc170051e..611c14c59bd3b 100644 --- a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config/replication.yaml +++ b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config/replication.yaml @@ -21,4 +21,4 @@ streams: sql: | select all_user_id, name from public."all_Users" - object: public.all_users # need to add 'object' key for custom SQL \ No newline at end of file + object: public.all_users # need to add 'object' key for custom SQL diff --git a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config_disabled/replication.yaml b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config_disabled/replication.yaml index 371f0b7fad181..91fc8c87acf25 100644 --- a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config_disabled/replication.yaml +++ b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_config_disabled/replication.yaml @@ -22,4 +22,4 @@ streams: sql: | select all_user_id, name from public."all_Users" - object: public.all_users # need to add 'object' key for custom SQL \ No newline at end of file + object: public.all_users # need to add 'object' key for custom SQL diff --git a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_with_meta_config/replication.yaml b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_with_meta_config/replication.yaml index 01cf4fcb5aa1a..e8ec9cbacd168 100644 --- a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_with_meta_config/replication.yaml +++ b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/base_with_meta_config/replication.yaml @@ -34,4 +34,4 @@ streams: env: SLING_LOADED_AT_COLUMN: true # adds the _sling_loaded_at timestamp column - SLING_STREAM_URL_COLUMN: true # if source is file, adds a _sling_stream_url column with file path / url \ No newline at end of file + SLING_STREAM_URL_COLUMN: true # if source is file, adds a _sling_stream_url column with file path / url diff --git a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/csv_to_sqlite_config/replication.yaml b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/csv_to_sqlite_config/replication.yaml index 3fc898330d877..b928af547823e 100644 --- a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/csv_to_sqlite_config/replication.yaml +++ b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/csv_to_sqlite_config/replication.yaml @@ -1,8 +1,8 @@ -source: SLING_FILE +source: SLING_FILE target: SLING_SQLITE defaults: mode: full-refresh object: "{stream_table}" -streams: {} \ No newline at end of file +streams: {} diff --git a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/README.md b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/README.md index c257e486a4b44..4df5fb1e4e799 100644 --- a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/README.md +++ b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/README.md @@ -1,7 +1,7 @@ This is some helper code for internally testing the integration with Sling. Run `make run` to build and run a simple Postgres instance that is fed with some -sample data. +sample data. Run the job in `sling_dag.py` with `dagster dev -f sling_dag.py` to see Dagster load the assets from the replication file, and sync data from PG to DuckDB using @@ -9,4 +9,4 @@ Sling. You can interact with the duckdb instance which defaults to /var/tmp/duckdb.db -This folder is not currently used for automated testing. \ No newline at end of file +This folder is not currently used for automated testing. diff --git a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/sling_replication.yaml b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/sling_replication.yaml index dc2324e33b4c5..bba66d9c4a156 100644 --- a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/sling_replication.yaml +++ b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/sling_replication.yaml @@ -2,16 +2,16 @@ source: MY_POSTGRES target: MY_DUCKDB defaults: - mode: full-refresh + mode: full-refresh - object: '{stream_schema}_{stream_table}' + object: "{stream_schema}_{stream_table}" streams: public.accounts: public.users: disabled: true public.finance_departments_old: - object: 'departments' # overwrite default object + object: "departments" # overwrite default object source_options: empty_as_null: false meta: @@ -21,7 +21,7 @@ streams: mode: incremental # overwrite default mode primary_key: id update_key: last_updated_at - + public.all_users: sql: | select all_user_id, name