[embedded-elt] add an example for the Sling Decorator

examples/experimental/sling_decorator/Makefile

@@ -0,0 +1,36 @@
+@PHONY: build run clean sync stop check_version
+
+check_version:
+	@required_version="1.1.7"; \
+	current_version=$$(sling --version | awk '{print $$2}'); \
+	if [ "$$(printf '%s\n' "$$required_version" "$$current_version" | sort -V | head -n1)" = "$$required_version" ]; then \
+		exit 0; \
+	else \
+		echo "sling version is less than $$required_version", run pip install --upgrade sling; \
+		exit 1; \
+	fi
+
+clean:
+	docker ps -q -f name=sling-postgres | xargs -r docker stop && docker ps -aq -f name=sling-postgres | xargs -r docker rm
+
+stop:
+	docker ps -q -f name=sling-postgres | xargs -r docker stop
+
+build:
+	docker build -t sling-postgres .
+
+run: build stop
+	docker run --name sling-postgres --detach -p 54321:5432 -ti --rm sling-postgres
+
+sync: check_version
+	MY_POSTGRES=postgres://postgres:postgres@localhost:54321/finance?sslmode=disable \
+				MY_DUCKDB=duckdb:///var/tmp/duckdb.db \
+				sling run -r sling_replication.yaml
+
+verify:	
+	@echo "		POSTGRES"
+	@echo "--------------------------------"
+	@echo "SELECT * FROM public.\"all_Users\";" | docker exec -i sling-postgres psql -U postgres -d finance
+	@echo "		DUCKDB"
+	@echo "--------------------------------"
+	@echo "SELECT * FROM public.all_users;" | duckdb /var/tmp/duckdb.db

examples/experimental/sling_decorator/README.md

@@ -0,0 +1,72 @@
+# (Experimental) Sling with Asset Decorators
+
+This is an example of how to use the new Sling `@sling_assets` decorator
+to sync data from Postgres to DuckDB.
+
+In order to run this example, we will setup a Docker instance of Postgres
+which some sample data to sync into DuckDB. The following steps will help you
+validate that you are able to run the example locally outside of Dagster using
+the included replication.yaml
+
+## Initial Setup
+
+First, we need to start the Postgres instance. The Makefile offers aliases
+to simplify building and running a Postgres instance:
+
+```bash
+make build  # builds the docker image
+make run    # runs the docker image
+```
+
+Next, validate that everything works locally:
+
+```bash
+pip install --upgrade sling
+make sync   # runs sling
+make verify
+```
+
+You should see output similar to this
+
+```
+                POSTGRES
+--------------------------------
+ all_user_id |      name
+-------------+----------------
+           1 | Alice Johnson
+           2 | Bob Williams
+           3 | Charlie Miller
+(3 rows)
+
+                DUCKDB
+--------------------------------
+┌─────────────┬────────────────┬──────────────────┐
+│ all_user_id │      name      │ _sling_loaded_at │
+│    int32    │    varchar     │      int32       │
+├─────────────┼────────────────┼──────────────────┤
+│           1 │ Alice Johnson  │       1708665267 │
+│           2 │ Bob Williams   │       1708665267 │
+│           3 │ Charlie Miller │       1708665267 │
+└─────────────┴────────────────┴──────────────────┘
+```
+
+## Getting started
+
+To download this example, run:
+
+```shell
+dagster project from-example --name my-dagster-project --example sling_decorator
+```
+
+To install this example and its dependencies, run:
+
+```shell
+cd my-dagster-project
+pip install -e .
+```
+
+Now run Dagster to load the sample Sling pipeline:
+
+```shell
+dagster dev
+```

examples/experimental/sling_decorator/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sling_decorator_example"
+version = "0.1.0"
+dependencies = [
+    "dagster-webserver","sling>=1.1.7"
+]
+[tool.setuptools]
+packages = ["sling_decorator"]
+
+[tool.dagster]
+module_name = "sling_decorator"

python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/replication_configs/pg_to_duckdb_with_dag_config/sling_dag.py → examples/experimental/sling_decorator/sling_decorator/__init__.py

@@ -1,15 +1,21 @@
 from dagster import Definitions, file_relative_path
-from dagster_embedded_elt.sling import DagsterSlingTranslator, sling_assets
-from dagster_embedded_elt.sling.resources import SlingConnectionResource, SlingResource
+from dagster_embedded_elt.sling import (
+    DagsterSlingTranslator,
+    sling_assets,
+)
+from dagster_embedded_elt.sling.resources import (
+    SlingConnectionResource,
+    SlingResource,
+)
 
-replication_config = file_relative_path(__file__, "sling_replication.yaml")
+replication_config = file_relative_path(__file__, "../sling_replication.yaml")
 
 sling_resource = SlingResource(
     connections=[
         SlingConnectionResource(
             name="MY_POSTGRES",
             type="postgres",
-            connection_string="postgres://postgres:postgres@localhost:5432/finance?sslmode=disable",
+            connection_string="postgres://postgres:postgres@localhost:54321/finance?sslmode=disable",
         ),
         SlingConnectionResource(
             name="MY_DUCKDB",
@@ -31,6 +37,10 @@ def my_assets(context, sling: SlingResource):
 
 
 defs = Definitions(
-    assets=[my_assets],
-    resources={"sling": sling_resource},
+    assets=[
+        my_assets,
+    ],
+    resources={
+        "sling": sling_resource,
+    },
 )