Skip to content

Commit

Permalink
[docs] Add tested snippets to embedded-elt
Browse files Browse the repository at this point in the history
## Summary & Motivation

This fixes some formatting issues in a Python code block and uses snippets for the code blocks in the example
  • Loading branch information
PedramNavid authored Oct 12, 2023
1 parent f46a70a commit 1983c75
Show file tree
Hide file tree
Showing 9 changed files with 146 additions and 13 deletions.
2 changes: 1 addition & 1 deletion docs/content/api/modules.json

Large diffs are not rendered by default.

58 changes: 47 additions & 11 deletions docs/content/integrations/embedded-elt.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -50,18 +50,19 @@ sling = SlingResource(source_connection=source, ...)

Note that no path is required in the source connection, as that is provided by the asset itself.

````python
```python
asset_def = build_sling_asset(
asset_spec=AssetSpec("my_file"),
source_stream=f"file://{path_to_file}",
...
)
```

For database connections, you can provide a connection string or a dictionary of keyword arguments. For example, to connect to a SQLite database, you can provide a path to the database using the `instance` keyword, which is specified in [Sling's SQLite connection](https://docs.slingdata.io/connections/database-connections/sqlite) documentation.

```python
source = SlingSourceConnection(type="sqlite", instance="path/to/sqlite.db")
````
```

---

Expand Down Expand Up @@ -107,29 +108,64 @@ sling_job = build_assets_job(

This is an example of how to setup a Sling sync between Postgres and Snowflake:

```python
```python file=/integrations/embedded_elt/postgres_snowflake.py
import os
from dagster_embedded_elt.sling import SlingResource, SlingSourceConnection, SlingTargetConnection

from dagster_embedded_elt.sling import (
SlingMode,
SlingResource,
SlingSourceConnection,
SlingTargetConnection,
build_sling_asset,
)

from dagster import AssetSpec

source = SlingSourceConnection(
type="postgres", host="localhost", port=5432, database="my_database",
user="my_user", password=os.getenv("PG_PASS")
type="postgres",
host="localhost",
port=5432,
database="my_database",
user="my_user",
password=os.getenv("PG_PASS"),
)

target = SlingTargetConnection(
type="snowflake", host="hostname.snowflake", user="username",
database="database", password=os.getenv("SF_PASSWORD"), role="role"
type="snowflake",
host="hostname.snowflake",
user="username",
database="database",
password=os.getenv("SF_PASSWORD"),
role="role",
)

sling = SlingResource(source_connection=source, target_connection=target)

asset_def = build_sling_asset(
asset_spec=AssetSpec("my_asset_name"),
source_stream="public.my_table",
target_object="marts.my_table",
mode=SlingMode.INCREMENTAL,
primary_key="id",
)
```

Similarily, you can define file/storage connections:

```python
```python startafter=start_storage_config endbefore=end_storage_config file=/integrations/embedded_elt/s3_snowflake.py
source = SlingSourceConnection(
type="s3", bucket="sling-bucket",
type="s3",
bucket="sling-bucket",
access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY")
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
)

sling = SlingResource(source_connection=source, target_connection=target)

asset_def = build_sling_asset(
asset_spec=AssetSpec("my_asset_name"),
source_stream="s3://my-bucket/my_file.parquet",
target_object="marts.my_table",
primary_key="id",
)
```
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# pyright: reportGeneralTypeIssues=none
# pyright: reportOptionalMemberAccess=none

import os

from dagster_embedded_elt.sling import (
SlingMode,
SlingResource,
SlingSourceConnection,
SlingTargetConnection,
build_sling_asset,
)

from dagster import AssetSpec

source = SlingSourceConnection(
type="postgres",
host="localhost",
port=5432,
database="my_database",
user="my_user",
password=os.getenv("PG_PASS"),
)

target = SlingTargetConnection(
type="snowflake",
host="hostname.snowflake",
user="username",
database="database",
password=os.getenv("SF_PASSWORD"),
role="role",
)

sling = SlingResource(source_connection=source, target_connection=target)

asset_def = build_sling_asset(
asset_spec=AssetSpec("my_asset_name"),
source_stream="public.my_table",
target_object="marts.my_table",
mode=SlingMode.INCREMENTAL,
primary_key="id",
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# pyright: reportGeneralTypeIssues=none
# pyright: reportOptionalMemberAccess=none

import os

from dagster_embedded_elt.sling import (
SlingResource,
SlingSourceConnection,
SlingTargetConnection,
build_sling_asset,
)

from dagster import AssetSpec

target = SlingTargetConnection(
type="snowflake",
host="hostname.snowflake",
user="username",
database="database",
password=os.getenv("SF_PASSWORD"),
role="role",
)


# start_storage_config
source = SlingSourceConnection(
type="s3",
bucket="sling-bucket",
access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
)

sling = SlingResource(source_connection=source, target_connection=target)

asset_def = build_sling_asset(
asset_spec=AssetSpec("my_asset_name"),
source_stream="s3://my-bucket/my_file.parquet",
target_object="marts.my_table",
primary_key="id",
)

# end_storage_config
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from docs_snippets.integrations.embedded_elt.postgres_snowflake import (
asset_def as asset_def_postgres,
)
from docs_snippets.integrations.embedded_elt.s3_snowflake import (
asset_def as asset_def_s3,
)


def test_asset_defs():
assert asset_def_postgres
assert asset_def_s3
1 change: 1 addition & 0 deletions examples/docs_snippets/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"dagster-dask",
"dagster-duckdb",
"dagster-duckdb-pandas",
"dagster-embedded-elt",
"dagster-fivetran",
"dagster-gcp",
"dagster-graphql",
Expand Down
1 change: 1 addition & 0 deletions examples/docs_snippets/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ deps =
-e ../../python_modules/libraries/dagster-dask
-e ../../python_modules/libraries/dagster-duckdb
-e ../../python_modules/libraries/dagster-duckdb-pandas
-e ../../python_modules/libraries/dagster-embedded-elt
-e ../../python_modules/libraries/dagster-fivetran
-e ../../python_modules/libraries/dagster-gcp
-e ../../python_modules/libraries/dagster-k8s
Expand Down

Large diffs are not rendered by default.

1 comment on commit 1983c75

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deploy preview for dagster-docs ready!

✅ Preview
https://dagster-docs-h71y1j3ob-elementl.vercel.app
https://master.dagster.dagster-docs.io

Built with commit 1983c75.
This pull request is being automatically deployed with vercel-action

Please sign in to comment.