From 9df8c5f7ed3f4314d089261a14fdd10ae49225e1 Mon Sep 17 00:00:00 2001 From: Yuhan Luo <4531914+yuhan@users.noreply.github.com> Date: Tue, 7 May 2024 10:46:46 -0700 Subject: [PATCH] docs nav RFC: move pipes to concepts (#20712) ## Summary & Motivation this PR moves pipes pages to Concepts - updates all internal links - updates nav -- open to suggestions to the exact order of the nav - sets up redirects so /guides/dagster-pipes URLs out there would still work ## How I Tested These Changes preview: https://03-25-docs-move-pipes-to-concepts.dagster.dagster-docs.io/concepts/dagster-pipes image --- docs/content/_apidocs.mdx | 2 +- docs/content/_navigation.json | 84 +++++++++---------- docs/content/concepts.mdx | 31 +++++++ .../{guides => concepts}/dagster-pipes.mdx | 10 +-- .../dagster-pipes/aws-lambda.mdx | 10 +-- ...agster-pipes-details-and-customization.mdx | 6 +- .../dagster-pipes/databricks.mdx | 10 +-- .../integrating-docker-with-dagster-pipes.mdx | 0 .../dagster-pipes/kubernetes.mdx | 10 +-- .../dagster-pipes/subprocess.mdx | 12 +-- .../subprocess/create-subprocess-asset.mdx | 8 +- .../subprocess/modify-external-code.mdx | 10 +-- .../dagster-pipes/subprocess/reference.mdx | 2 +- docs/content/guides.mdx | 10 --- docs/next/util/redirectUrls.json | 9 +- 15 files changed, 120 insertions(+), 94 deletions(-) rename docs/content/{guides => concepts}/dagster-pipes.mdx (91%) rename docs/content/{guides => concepts}/dagster-pipes/aws-lambda.mdx (96%) rename docs/content/{guides => concepts}/dagster-pipes/dagster-pipes-details-and-customization.mdx (97%) rename docs/content/{guides => concepts}/dagster-pipes/databricks.mdx (97%) rename docs/content/{guides => concepts}/dagster-pipes/integrating-docker-with-dagster-pipes.mdx (100%) rename docs/content/{guides => concepts}/dagster-pipes/kubernetes.mdx (95%) rename docs/content/{guides => concepts}/dagster-pipes/subprocess.mdx (66%) rename docs/content/{guides => concepts}/dagster-pipes/subprocess/create-subprocess-asset.mdx (90%) rename docs/content/{guides => concepts}/dagster-pipes/subprocess/modify-external-code.mdx (95%) rename docs/content/{guides => concepts}/dagster-pipes/subprocess/reference.mdx (99%) diff --git a/docs/content/_apidocs.mdx b/docs/content/_apidocs.mdx index a52d99041fdca..6f3c6d6dd171a 100644 --- a/docs/content/_apidocs.mdx +++ b/docs/content/_apidocs.mdx @@ -206,7 +206,7 @@ APIs from the core `dagster` package, divided roughly by topic: APIs for working with the{" "} - Dagster Pipes protocol from the + Dagster Pipes protocol from the orchestration side. diff --git a/docs/content/_navigation.json b/docs/content/_navigation.json index 673201184fb0d..0458981967aae 100644 --- a/docs/content/_navigation.json +++ b/docs/content/_navigation.json @@ -305,6 +305,46 @@ } ] }, + { + "title": "Dagster Pipes (Experimental)", + "path": "/concepts/dagster-pipes", + "children": [ + { + "title": "Dagster Pipes tutorial", + "path": "/concepts/dagster-pipes/subprocess", + "children": [ + { + "title": "Part 1: Define a Dagster asset", + "path": "/concepts/dagster-pipes/subprocess/create-subprocess-asset" + }, + { + "title": "Part 2: Modify external code", + "path": "/concepts/dagster-pipes/subprocess/modify-external-code" + }, + { + "title": "Dagster Pipes + subprocess reference", + "path": "/concepts/dagster-pipes/subprocess/reference" + } + ] + }, + { + "title": "Dagster Pipes + AWS Lambda", + "path": "/concepts/dagster-pipes/aws-lambda" + }, + { + "title": "Dagster Pipes + Databricks", + "path": "/concepts/dagster-pipes/databricks" + }, + { + "title": "Dagster Pipes + Kubernetes", + "path": "/concepts/dagster-pipes/kubernetes" + }, + { + "title": "Details and customization", + "path": "/concepts/dagster-pipes/dagster-pipes-details-and-customization" + } + ] + }, { "title": "I/O management", "children": [ @@ -685,7 +725,7 @@ "path": "/dagster-plus/managing-deployments/branch-deployments", "children": [ { - "title": "Overview", + "title": "Overview", "path": "/dagster-plus/managing-deployments/branch-deployments" }, { @@ -701,7 +741,7 @@ "path": "/dagster-plus/managing-deployments/branch-deployments/using-branch-deployments" }, { - "title": "Change Tracking", + "title": "Change Tracking", "path": "/dagster-plus/managing-deployments/branch-deployments/change-tracking" } ] @@ -1057,46 +1097,6 @@ } ] }, - { - "title": "Dagster Pipes (Experimental)", - "path": "/guides/dagster-pipes", - "children": [ - { - "title": "Dagster Pipes tutorial", - "path": "/guides/dagster-pipes/subprocess", - "children": [ - { - "title": "Part 1: Define a Dagster asset", - "path": "/guides/dagster-pipes/subprocess/create-subprocess-asset" - }, - { - "title": "Part 2: Modify external code", - "path": "/guides/dagster-pipes/subprocess/modify-external-code" - }, - { - "title": "Dagster Pipes & subprocess reference", - "path": "/guides/dagster-pipes/subprocess/reference" - } - ] - }, - { - "title": "Dagster Pipes & AWS Lambda", - "path": "/guides/dagster-pipes/aws-lambda" - }, - { - "title": "Dagster Pipes & Databricks", - "path": "/guides/dagster-pipes/databricks" - }, - { - "title": "Dagster Pipes & Kubernetes", - "path": "/guides/dagster-pipes/kubernetes" - }, - { - "title": "Details and customization", - "path": "/guides/dagster-pipes/dagster-pipes-details-and-customization" - } - ] - }, { "title": "Version migration", "path": "/migration", diff --git a/docs/content/concepts.mdx b/docs/content/concepts.mdx index bbc1540b1a69e..8514cecd1c01a 100644 --- a/docs/content/concepts.mdx +++ b/docs/content/concepts.mdx @@ -211,6 +211,37 @@ Apply tags and metadata to organize your project and provide useful context to o > +### Dagster Pipes (Experimental) + +Dagster Pipes is a toolkit for building integrations between Dagster and external execution environments. + + + + + + + + + + ### I/O management I/O managers are user-provided objects that store asset and op outputs and load them as inputs to downstream assets and ops. diff --git a/docs/content/guides/dagster-pipes.mdx b/docs/content/concepts/dagster-pipes.mdx similarity index 91% rename from docs/content/guides/dagster-pipes.mdx rename to docs/content/concepts/dagster-pipes.mdx index a4d7134d34558..e3905ea19d06f 100644 --- a/docs/content/guides/dagster-pipes.mdx +++ b/docs/content/concepts/dagster-pipes.mdx @@ -74,12 +74,12 @@ While Dagster Pipes is lightweight and flexible, there are a few limitations to Ready to get started with Dagster Pipes? Depending on what your goal is, how you approach using Dagster Pipes can differ. -- **If you’re writing scripts that are being orchestrated**, all you’ll need to do is lightly modify your existing scripts. Check out the [Dagster Pipes tutorial](/guides/dagster-pipes/subprocess) to get up and running. +- **If you’re writing scripts that are being orchestrated**, all you’ll need to do is lightly modify your existing scripts. Check out the [Dagster Pipes tutorial](/concepts/dagster-pipes/subprocess) to get up and running. - **If you need to orchestrate scripts others have written**, you can use our ready-to-go integrations to execute scripts in your chosen technology: - - [Amazon Web Services Lambda](/guides/dagster-pipes/aws-lambda) - - [Databricks](/guides/dagster-pipes/databricks) - - [Kubernetes](/guides/dagster-pipes/kubernetes) + - [Amazon Web Services Lambda](/concepts/dagster-pipes/aws-lambda) + - [Databricks](/concepts/dagster-pipes/databricks) + - [Kubernetes](/concepts/dagster-pipes/kubernetes) -- **If you don’t see your integration or you want to fully customize your Pipes experience**, check out the [Dagster Pipes details and customization guide](/guides/dagster-pipes/dagster-pipes-details-and-customization) to learn how to create a custom experience. +- **If you don’t see your integration or you want to fully customize your Pipes experience**, check out the [Dagster Pipes details and customization guide](/concepts/dagster-pipes/dagster-pipes-details-and-customization) to learn how to create a custom experience. diff --git a/docs/content/guides/dagster-pipes/aws-lambda.mdx b/docs/content/concepts/dagster-pipes/aws-lambda.mdx similarity index 96% rename from docs/content/guides/dagster-pipes/aws-lambda.mdx rename to docs/content/concepts/dagster-pipes/aws-lambda.mdx index 210c6cec86a4a..33b54f02ffea6 100644 --- a/docs/content/guides/dagster-pipes/aws-lambda.mdx +++ b/docs/content/concepts/dagster-pipes/aws-lambda.mdx @@ -8,13 +8,13 @@ description: "Learn to integrate Dagster Pipes with AWS Lambda to launch externa Heads up! This guide focuses on using an out-of-the-box Amazon Web Services (AWS) Lambda resource. For further customization, use the{" "} - + open_pipes_session approach {" "} instead. -In this guide, we’ll show you how to use [Dagster Pipes](/guides/dagster-pipes) with Dagster’s AWS Lambda integration to invoke a Lambda function and execute external code. +In this guide, we’ll show you how to use [Dagster Pipes](/concepts/dagster-pipes) with Dagster’s AWS Lambda integration to invoke a Lambda function and execute external code. Pipes allows your code to interact with Dagster outside of a full Dagster environment. Instead, the environment only needs to contain `dagster-pipes`, a single-file Python package with no dependencies that can be installed from PyPI or easily vendored. `dagster-pipes` handles streaming `stdout`/`stderr` and Dagster events back to the orchestration process. @@ -154,7 +154,7 @@ Let's review what this code does: We're using the default context loader () and message writer () in this example. These objects establish communication between the orchestration and external process. On the orchestration end, these match a corresponding `PipesLambdaEventContextInjector` and `PipesLambdaLogsMessageReader`, which are instantiated inside the . -- **Inside the body of the context manager (), retrieve a log and report an asset materialization.** These calls use the temporary communications channels established by and . To see the full range of what you can do with the , see the API docs or the general [Pipes documentation](/guides/dagster-pipes). +- **Inside the body of the context manager (), retrieve a log and report an asset materialization.** These calls use the temporary communications channels established by and . To see the full range of what you can do with the , see the API docs or the general [Pipes documentation](/concepts/dagster-pipes). At this point you can execute the rest of your AWS Lambda code as normal, invoking various APIs as needed. @@ -257,11 +257,11 @@ In this step, you’ll invoke the AWS Lambda function you defined in [Step 1](#s , ), others will need a greater level of control over Pipes. This is particularly the case for users seeking to connect large existing codebases to Dagster. +[Dagster Pipes](/concepts/dagster-pipes) is a toolkit for integrating Dagster with an arbitrary external compute environment. While many users will be well-served by the simplified interface offered by Pipes client objects (e.g. , ), others will need a greater level of control over Pipes. This is particularly the case for users seeking to connect large existing codebases to Dagster. This guide will cover the lower level Pipes APIs and how you can compose them to provide a custom solution for your data platform. @@ -487,10 +487,10 @@ class MyCustomCloudServiceMessageWriterChannel(PipesBlobStoreMessageWriterChanne diff --git a/docs/content/guides/dagster-pipes/databricks.mdx b/docs/content/concepts/dagster-pipes/databricks.mdx similarity index 97% rename from docs/content/guides/dagster-pipes/databricks.mdx rename to docs/content/concepts/dagster-pipes/databricks.mdx index f8c0b7d7e37ad..9f48ab361525c 100644 --- a/docs/content/guides/dagster-pipes/databricks.mdx +++ b/docs/content/concepts/dagster-pipes/databricks.mdx @@ -5,7 +5,7 @@ description: "Learn to integrate Dagster Pipes with Databricks to launch externa # Integrating Databricks with Dagster Pipes -In this guide, we’ll show you how to use [Dagster Pipes](/guides/dagster-pipes) with Dagster’s Databricks integration to launch Databricks jobs. +In this guide, we’ll show you how to use [Dagster Pipes](/concepts/dagster-pipes) with Dagster’s Databricks integration to launch Databricks jobs. Pipes allows your Databricks jobs to stream logs (including `stdout` and `stderr` of the driver process) and events back to Dagster. This does not require a full Dagster environment on Databricks; instead: @@ -210,13 +210,13 @@ with open_dagster_pipes( Before we go any futher, let's review what this script does: -- **Imports `PipesDbfsContextLoader`, `PipesDbfsMessageWriter`, and `open_dagster_pipes` from `dagster_pipes`.** The and are DBFS-specific implementations of the and . Refer to the [Dagster Pipes details and customization Guide](/guides/dagster-pipes/dagster-pipes-details-and-customization) for protocol details. +- **Imports `PipesDbfsContextLoader`, `PipesDbfsMessageWriter`, and `open_dagster_pipes` from `dagster_pipes`.** The and are DBFS-specific implementations of the and . Refer to the [Dagster Pipes details and customization Guide](/concepts/dagster-pipes/dagster-pipes-details-and-customization) for protocol details. Both objects write temporary files on DBFS for communication between the orchestration and external process. The and match a corresponding `PipesDbfsContextInjector` and `PipesDbfsMessageReader` on the orchestration end, which are instantiated inside the . - **Passes the context loader and message writer to the context manager**, which yields an instance of called `pipes`. - Inside the body of the context manager are various calls against `pipes` to retrieve an extra, log, and report an asset materialization. All of these calls will use the DBFS temporary file-based communications channels established by and . To see the full range of what you can do with the , see the API docs or the general [Pipes guide](/guides/dagster-pipes). + Inside the body of the context manager are various calls against `pipes` to retrieve an extra, log, and report an asset materialization. All of these calls will use the DBFS temporary file-based communications channels established by and . To see the full range of what you can do with the , see the API docs or the general [Pipes guide](/concepts/dagster-pipes). At this point you can execute the rest of your Databricks code as normal, invoking various APIs as needed. @@ -378,11 +378,11 @@ def databricks_asset(context: AssetExecutionContext): Heads up! This guide focuses on using an out-of-the-box Kubernetes resource. For further customization, use the{" "} - + open_pipes_session approach {" "} instead. -In this guide, we’ll show you how to use [Dagster Pipes](/guides/dagster-pipes) with Dagster’s Kubernetes integration to launch Kubernetes pods and execute external code. +In this guide, we’ll show you how to use [Dagster Pipes](/concepts/dagster-pipes) with Dagster’s Kubernetes integration to launch Kubernetes pods and execute external code. Pipes allows your code to interact with Dagster outside of a full Dagster environment. Instead, the environment only needs to contain `dagster-pipes`, a single-file Python package with no dependencies that can be installed from PyPI or easily vendored. `dagster-pipes` handles streaming `stdout`/`stderr` and Dagster events back to the orchestration process. @@ -77,7 +77,7 @@ Let's review what this code does: We're using the default context loader () and message writer () in this example. These objects establish communication between the orchestration and external process. On the orchestration end, these match a corresponding `PipesContextInjector` and `PipesMessageReader`, which are instantiated inside the . -- **Inside the body of the context manager (), retrieve a log and report an asset materialization.** These calls use the temporary communications channels established by and . To see the full range of what you can do with the , see the API docs or the general [Pipes documentation](/guides/dagster-pipes). +- **Inside the body of the context manager (), retrieve a log and report an asset materialization.** These calls use the temporary communications channels established by and . To see the full range of what you can do with the , see the API docs or the general [Pipes documentation](/concepts/dagster-pipes). At this point you can execute the rest of your Kubernetes code as normal, invoking various APIs as needed. @@ -237,11 +237,11 @@ In this step, you’ll run the Kubernetes container you defined in [Step 1](#ste @@ -19,7 +19,7 @@ This guide focuses on using an out-of-the-box `PipesSubprocessClient` resource. - + @@ -51,4 +51,4 @@ if __name__ == "__main__": ## Ready to get started? -When you've fulfilled all the prerequisites for the tutorial, you can get started by [creating a Dagster asset that executes a subprocess](/guides/dagster-pipes/subprocess/create-subprocess-asset). +When you've fulfilled all the prerequisites for the tutorial, you can get started by [creating a Dagster asset that executes a subprocess](/concepts/dagster-pipes/subprocess/create-subprocess-asset). diff --git a/docs/content/guides/dagster-pipes/subprocess/create-subprocess-asset.mdx b/docs/content/concepts/dagster-pipes/subprocess/create-subprocess-asset.mdx similarity index 90% rename from docs/content/guides/dagster-pipes/subprocess/create-subprocess-asset.mdx rename to docs/content/concepts/dagster-pipes/subprocess/create-subprocess-asset.mdx index f9f0f0f1d6373..dcfb0628178ca 100644 --- a/docs/content/guides/dagster-pipes/subprocess/create-subprocess-asset.mdx +++ b/docs/content/concepts/dagster-pipes/subprocess/create-subprocess-asset.mdx @@ -7,7 +7,7 @@ description: "Learn how to create a Dagster asset that invokes a subprocess that -This is part one of the [Using Dagster Pipes](/guides/dagster-pipes/subprocess) tutorial. If you are looking for how to modify your existing code that is already being orchestrated by Dagster, you can jump to Part 2: Modify external code. +This is part one of the [Using Dagster Pipes](/concepts/dagster-pipes/subprocess) tutorial. If you are looking for how to modify your existing code that is already being orchestrated by Dagster, you can jump to Part 2: Modify external code. @@ -17,7 +17,7 @@ In this part of the tutorial, you'll create a Dagster asset that, in its executi ## Step 1: Define the Dagster asset -Before getting started, make sure you have fulfilled all the [prerequisites](/guides/dagster-pipes/subprocess#prerequisites) for the tutorial. You should have a standalone Python script named `external_code.py` which looks like the following: +Before getting started, make sure you have fulfilled all the [prerequisites](/concepts/dagster-pipes/subprocess#prerequisites) for the tutorial. You should have a standalone Python script named `external_code.py` which looks like the following: ```python file=/guides/dagster/dagster_pipes/subprocess/part_1/external_code.py lines=2- import pandas as pd @@ -35,7 +35,7 @@ if __name__ == "__main__": ### Step 1.1: Define the asset -First, create a new file named `dagster_code.py` in the same directory as the `external_code.py` file you created earlier in the [Prerequisites](/guides/dagster-pipes/subprocess#prerequisites) step. +First, create a new file named `dagster_code.py` in the same directory as the `external_code.py` file you created earlier in the [Prerequisites](/concepts/dagster-pipes/subprocess#prerequisites) step. Next, you’ll define the asset. Copy and paste the following into the file: @@ -200,4 +200,4 @@ In this step, you’ll execute the subprocess asset you created in earlier steps ## What's next? -At this point, you've created a Dagster asset that invokes an external Python script, launched the code in a subprocess, and viewed the result in Dagster UI. Next, you'll learn how to [modify your external code to work with Dagster Pipes](/guides/dagster-pipes/subprocess/modify-external-code) to send information back to Dagster. +At this point, you've created a Dagster asset that invokes an external Python script, launched the code in a subprocess, and viewed the result in Dagster UI. Next, you'll learn how to [modify your external code to work with Dagster Pipes](/concepts/dagster-pipes/subprocess/modify-external-code) to send information back to Dagster. diff --git a/docs/content/guides/dagster-pipes/subprocess/modify-external-code.mdx b/docs/content/concepts/dagster-pipes/subprocess/modify-external-code.mdx similarity index 95% rename from docs/content/guides/dagster-pipes/subprocess/modify-external-code.mdx rename to docs/content/concepts/dagster-pipes/subprocess/modify-external-code.mdx index 4b68c570cccd1..4b66ba247ba28 100644 --- a/docs/content/guides/dagster-pipes/subprocess/modify-external-code.mdx +++ b/docs/content/concepts/dagster-pipes/subprocess/modify-external-code.mdx @@ -7,7 +7,7 @@ description: "With Dagster Pipes, you can incorporate existing code into Dagster -This is part two of the [Using Dagster Pipes](/guides/dagster-pipes/subprocess) tutorial. +This is part two of the [Using Dagster Pipes](/concepts/dagster-pipes/subprocess) tutorial. @@ -16,7 +16,7 @@ At this point, you should have two files: - `external_code.py` which is a standalone Python script that you want to orchestrate with Dagster. - `dagster_code.py` which includes a Dagster asset and other Dagster definitions. -In this section, you'll learn how to modify the standalone Python script to work with [Dagster Pipes](/guides/dagster-pipes) in order to stream information back to Dagster. To do this, you'll: +In this section, you'll learn how to modify the standalone Python script to work with [Dagster Pipes](/concepts/dagster-pipes) in order to stream information back to Dagster. To do this, you'll: - [Make Dagster context available in external code](#step-1-make-dagster-context-available-in-external-code) - [Stream log messages back to Dagster](#step-2-send-log-messages-to-dagster) @@ -318,9 +318,9 @@ In this tutorial, you learned how to get access to Dagster Pipes context, report What's next? From here, you can: -- Learn about other capabilities of executing external code in subprocess via Dagster Pipes in the [Subprocess reference](/guides/dagster-pipes/subprocess/reference) -- Learn how to [customize your own Dagster Pipes protocols](/guides/dagster-pipes/dagster-pipes-details-and-customization) +- Learn about other capabilities of executing external code in subprocess via Dagster Pipes in the [Subprocess reference](/concepts/dagster-pipes/subprocess/reference) +- Learn how to [customize your own Dagster Pipes protocols](/concepts/dagster-pipes/dagster-pipes-details-and-customization) - + diff --git a/docs/content/guides/dagster-pipes/subprocess/reference.mdx b/docs/content/concepts/dagster-pipes/subprocess/reference.mdx similarity index 99% rename from docs/content/guides/dagster-pipes/subprocess/reference.mdx rename to docs/content/concepts/dagster-pipes/subprocess/reference.mdx index aa769321e9fa7..9aaa62150ad53 100644 --- a/docs/content/guides/dagster-pipes/subprocess/reference.mdx +++ b/docs/content/concepts/dagster-pipes/subprocess/reference.mdx @@ -5,7 +5,7 @@ description: "This page shows ways to execute external code with Dagster Pipes w # Dagster Pipes subprocess reference -This reference shows usage of Dagster Pipes with other entities in the Dagster system. For a step-by-step walkthrough, refer to the [Dagster Pipes tutorial](/guides/dagster-pipes/subprocess). +This reference shows usage of Dagster Pipes with other entities in the Dagster system. For a step-by-step walkthrough, refer to the [Dagster Pipes tutorial](/concepts/dagster-pipes/subprocess). --- diff --git a/docs/content/guides.mdx b/docs/content/guides.mdx index b72347be00106..66836d8049f6a 100644 --- a/docs/content/guides.mdx +++ b/docs/content/guides.mdx @@ -46,16 +46,6 @@ Learn to apply [Dagster concepts](/concepts) to your work, explore experimental --- -## Dagster Pipes (Experimental) - -- [Dagster Pipes](/guides/dagster-pipes) - A high-level look at Dagster Pipes, a toolkit for building integrations between Dagster and external execution environments - -- [Dagster Pipes tutorial](/guides/dagster-pipes/subprocess) - Get started using Dagster Pipes with this tutorial, where you'll use a subprocess to execute code in an external environment - -- [Dagster Pipes details and customization](/guides/dagster-pipes/dagster-pipes-details-and-customization) - Learn about Dagster Pipes APIs and how to compose them to create a custom solution for your data platform - ---- - ## Migrating to Dagster - [Migrating from Airflow to Dagster](/integrations/airflow/migrating-to-dagster) - Perform a lift-and-shift migration from Airflow to Dagster diff --git a/docs/next/util/redirectUrls.json b/docs/next/util/redirectUrls.json index 717e810e2808f..d9d85dd4fd2ea 100644 --- a/docs/next/util/redirectUrls.json +++ b/docs/next/util/redirectUrls.json @@ -90,8 +90,8 @@ "statusCode": 302 }, { - "source": "/overview", - "destination": "/concepts", + "source": "/overview", + "destination": "/concepts", "statusCode": 302 }, { @@ -1083,5 +1083,10 @@ "source": "/dagster-cloud", "destination": "/dagster-plus", "statusCode": 302 + }, + { + "source": "/guides/dagster-pipes/:slug*", + "destination": "/concepts/dagster-pipes/:slug*", + "statusCode": 302 } ]