diff --git a/docs/docs-beta/.gitignore b/docs/docs-beta/.gitignore index 947019b9e7a8f..5082c18b1edcb 100644 --- a/docs/docs-beta/.gitignore +++ b/docs/docs-beta/.gitignore @@ -3,6 +3,7 @@ # Production /build +!/docs/guides/build # Generated files .docusaurus diff --git a/docs/docs-beta/README.md b/docs/docs-beta/README.md index dc02dd3a0d734..e42fa9148a661 100644 --- a/docs/docs-beta/README.md +++ b/docs/docs-beta/README.md @@ -15,8 +15,6 @@ To install dependencies: yarn install ``` -**Note**: The yarn binary is checked in, so you do not need to install yarn yourself. - It also uses [Vale](https://vale.sh/) to check for issues in the documentation. Install Vale with: @@ -44,7 +42,6 @@ The docs are broken down into the following sections: - [Tutorials](./docs/tutorials/) - [Guides](./docs/guides/) -- [Concepts](./docs/concepts/) `sidebar.ts` and `docusaurus.config.ts` are the main configuration files for the documentation. @@ -116,7 +113,7 @@ To build the site for production: yarn build ``` -This command generates static content into the `build` directory and can be served using any static contents hosting service. This also checks for any broken links in the documentation. +This command generates static content into the `build` directory and can be served using any static contents hosting service. This also checks for any broken links in the documentation. Note that you will need to store Algolia credentials in local environment variables to build the site for production. ## Deployment diff --git a/docs/docs-beta/docs/api/index.mdx b/docs/docs-beta/docs/api/index.mdx index 5bc60863ec197..dd34e2525d2ea 100644 --- a/docs/docs-beta/docs/api/index.mdx +++ b/docs/docs-beta/docs/api/index.mdx @@ -1,6 +1,6 @@ --- sidebar_class_name: hidden -title: API Docs +title: API reference --- import DocCardList from '@theme/DocCardList'; diff --git a/docs/docs-beta/docs/concepts/assets.md b/docs/docs-beta/docs/concepts/assets.md deleted file mode 100644 index 9ff5ed162de1f..0000000000000 --- a/docs/docs-beta/docs/concepts/assets.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Assets -unlisted: true ---- - -# Assets - -## Assets and ops - -Assets and ops are two different concepts in Dagster. diff --git a/docs/docs-beta/docs/concepts/assets/asset-checks.md b/docs/docs-beta/docs/concepts/assets/asset-checks.md deleted file mode 100644 index 328dac16243a4..0000000000000 --- a/docs/docs-beta/docs/concepts/assets/asset-checks.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Asset checks" -sidebar_position: 70 -unlisted: true ---- - -# Asset checks diff --git a/docs/docs-beta/docs/concepts/assets/asset-dependencies.md b/docs/docs-beta/docs/concepts/assets/asset-dependencies.md deleted file mode 100644 index 2089ba4680d50..0000000000000 --- a/docs/docs-beta/docs/concepts/assets/asset-dependencies.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Asset dependencies" -sidebar_position: 30 -unlisted: true ---- - -# Asset dependencies diff --git a/docs/docs-beta/docs/concepts/assets/thinking-in-assets.md b/docs/docs-beta/docs/concepts/assets/thinking-in-assets.md deleted file mode 100644 index fbbbcde65fe96..0000000000000 --- a/docs/docs-beta/docs/concepts/assets/thinking-in-assets.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Think in assets" -sidebar_position: 10 -unlisted: true ---- - -# Think in assets diff --git a/docs/docs-beta/docs/concepts/automation/declarative-automation.md b/docs/docs-beta/docs/concepts/automation/declarative-automation.md deleted file mode 100644 index 1e7221492f11c..0000000000000 --- a/docs/docs-beta/docs/concepts/automation/declarative-automation.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Declarative Automation" -sidebar_position: 10 -unlisted: true ---- - -# Declarative Automation diff --git a/docs/docs-beta/docs/concepts/automation/schedules.md b/docs/docs-beta/docs/concepts/automation/schedules.md deleted file mode 100644 index 25867d5cbee78..0000000000000 --- a/docs/docs-beta/docs/concepts/automation/schedules.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -unlisted: true ---- diff --git a/docs/docs-beta/docs/concepts/automation/sensors.md b/docs/docs-beta/docs/concepts/automation/sensors.md deleted file mode 100644 index 6ede003072419..0000000000000 --- a/docs/docs-beta/docs/concepts/automation/sensors.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Sensors" -sidebar_position: 20 -unlisted: true ---- - -# Sensors diff --git a/docs/docs-beta/docs/concepts/execution.md b/docs/docs-beta/docs/concepts/execution.md deleted file mode 100644 index 01f5affa3b030..0000000000000 --- a/docs/docs-beta/docs/concepts/execution.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -unlisted: true ---- - -# Execution diff --git a/docs/docs-beta/docs/concepts/io-managers.md b/docs/docs-beta/docs/concepts/io-managers.md deleted file mode 100644 index d9494abca542b..0000000000000 --- a/docs/docs-beta/docs/concepts/io-managers.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "I/O managers" -unlisted: true ---- diff --git a/docs/docs-beta/docs/concepts/ops-jobs.md b/docs/docs-beta/docs/concepts/ops-jobs.md deleted file mode 100644 index 4382fa8efa15d..0000000000000 --- a/docs/docs-beta/docs/concepts/ops-jobs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -unlisted: true ---- - -# Ops and jobs diff --git a/docs/docs-beta/docs/concepts/partitions.md b/docs/docs-beta/docs/concepts/partitions.md deleted file mode 100644 index c6219e4be7878..0000000000000 --- a/docs/docs-beta/docs/concepts/partitions.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Partitions" -unlisted: true ---- - -# Partitions diff --git a/docs/docs-beta/docs/concepts/resources.md b/docs/docs-beta/docs/concepts/resources.md deleted file mode 100644 index 08cdb3b01d061..0000000000000 --- a/docs/docs-beta/docs/concepts/resources.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -unlisted: true ---- - -# Resources diff --git a/docs/docs-beta/docs/concepts/schedules.mdx b/docs/docs-beta/docs/concepts/schedules.mdx deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docs/docs-beta/docs/concepts/sensors.md b/docs/docs-beta/docs/concepts/sensors.md deleted file mode 100644 index 25867d5cbee78..0000000000000 --- a/docs/docs-beta/docs/concepts/sensors.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -unlisted: true ---- diff --git a/docs/docs-beta/docs/concepts/understanding-assets.md b/docs/docs-beta/docs/concepts/understanding-assets.md deleted file mode 100644 index b19ef79420905..0000000000000 --- a/docs/docs-beta/docs/concepts/understanding-assets.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: About assets -description: Understanding the concept of assets in Dagster -last_update: - date: 2024-08-11 - author: Pedram Navid -unlisted: true ---- diff --git a/docs/docs-beta/docs/dagster-plus.md b/docs/docs-beta/docs/dagster-plus.md deleted file mode 100644 index 61c264492f0eb..0000000000000 --- a/docs/docs-beta/docs/dagster-plus.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Dagster+" -displayed_sidebar: "dagsterPlus" ---- - -# Dagster+ diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication/scim-provisioning.md b/docs/docs-beta/docs/dagster-plus/access/authentication/scim-provisioning.md deleted file mode 100644 index 6e45a0ddb65f0..0000000000000 --- a/docs/docs-beta/docs/dagster-plus/access/authentication/scim-provisioning.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: 'Utilizing SCIM provisioning' -displayed_sidebar: 'dagsterPlus' -unlisted: true ---- - -# Utilizing SCIM provisioning diff --git a/docs/docs-beta/docs/dagster-plus/access/rbac.md b/docs/docs-beta/docs/dagster-plus/access/rbac.md deleted file mode 100644 index 32098f061b1a3..0000000000000 --- a/docs/docs-beta/docs/dagster-plus/access/rbac.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Role-based Access Control" -displayed_sidebar: "dagsterPlus" -unlisted: true ---- - -# Role-based Access Control diff --git a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments.md b/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments.md deleted file mode 100644 index 004dd178c1c7e..0000000000000 --- a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: "Branch Deployments (CI)" -displayed_sidebar: "dagsterPlus" -sidebar_position: 3 -unlisted: true ---- - -# Branch Deployments (CI) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/change-tracking.md b/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/change-tracking.md deleted file mode 100644 index 77a183daf1ab4..0000000000000 --- a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/change-tracking.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Change Tracking in Branch Deployments" -displayed_sidebar: "dagsterPlus" -sidebar_position: 4 -sidebar_label: "Change Tracking" -unlisted: true ---- - -# Change Tracking in Branch Deployments diff --git a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/dagster-cloud-cli.md b/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/dagster-cloud-cli.md deleted file mode 100644 index 5794c59abf1ae..0000000000000 --- a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/dagster-cloud-cli.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Branch Deployments & the dagster-cloud CLI" -displayed_sidebar: "dagsterPlus" -sidebar_position: 3 -sidebar_label: "dagster-cloud CLI" -unlisted: true ---- - -# Use Branch Deployments with the dagster-cloud CLI diff --git a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/testing.md b/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/testing.md deleted file mode 100644 index ccd134eb8eb2b..0000000000000 --- a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/testing.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "Testing against production with Branch Deployments" -unlisted: true ---- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md index 932f32d19d1b7..f7e080d785acb 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md @@ -1,7 +1,7 @@ --- title: "Code location history and rollbacks" displayed_sidebar: "dagsterPlus" -sidebar_position: 4 +sidebar_position: 100 sidebar_label: "Code location history and rollbacks" --- @@ -54,4 +54,4 @@ If you notice an issue with newly deployed code, or your code fails to deploy su ## Next steps - Learn more about [Code Locations](/dagster-plus/deployment/code-locations) -- Learn how to [Alert when a code location fails to load](/dagster-plus/deployment/alerts#alerting-when-a-code-location-fails-to-load) +- Learn how to [Alert when a code location fails to load](/dagster-plus/features/alerts#alerting-when-a-code-location-fails-to-load) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-locations.md b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/index.md similarity index 99% rename from docs/docs-beta/docs/dagster-plus/deployment/code-locations.md rename to docs/docs-beta/docs/dagster-plus/deployment/code-locations/index.md index 093e6ee4391ab..ad60593d0bfb4 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-locations.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/index.md @@ -1,6 +1,7 @@ --- title: "Code locations" displayed_sidebar: "dagsterPlus" +sidebar_position: 50 --- # Code locations diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md b/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md index 4de13201cdd03..1fba813927a18 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md @@ -2,6 +2,7 @@ title: 'Dagster+ code requirements' displayed_sidebar: 'dagsterPlus' sidebar_label: "Code requirements" +sidebar_position: 100 --- Your Dagster project must meet a few requirements to run in Dagster+. diff --git a/docs/docs-beta/docs/dagster-plus/settings.md b/docs/docs-beta/docs/dagster-plus/deployment/dagster-plus-settings.md similarity index 76% rename from docs/docs-beta/docs/dagster-plus/settings.md rename to docs/docs-beta/docs/dagster-plus/deployment/dagster-plus-settings.md index a7c4dad36b30e..0d7330cf5a6a9 100644 --- a/docs/docs-beta/docs/dagster-plus/settings.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/dagster-plus-settings.md @@ -1,6 +1,7 @@ --- title: "Dagster+ settings" unlisted: true +sidebar_position: 200 --- # Dagster+ settings diff --git a/docs/docs-beta/docs/dagster-plus/deployment/settings.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-settings.md similarity index 64% rename from docs/docs-beta/docs/dagster-plus/deployment/settings.md rename to docs/docs-beta/docs/dagster-plus/deployment/deployment-settings.md index 355740e8e2e97..bbfa62beba985 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/settings.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-settings.md @@ -1,8 +1,9 @@ --- title: "Deployment settings" displayed_sidebar: "dagsterPlus" -sidebar_label: "Settings" +sidebar_label: "Deployment settings" unlisted: true +sidebar_position: 300 --- # Deployment settings diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types.md index f8d3d29a1f9a6..383e5b696c920 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types.md @@ -1,5 +1,7 @@ --- +title: Deployment types unlisted: true +sidebar_position: 400 --- -## Placeholder +# Deployment types diff --git a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/agent-config.md b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/agent-config.md index 5fabc3547076b..a22abdf31e07b 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/agent-config.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/agent-config.md @@ -1,7 +1,7 @@ --- title: "Set environment variables using agent config" displayed_sidebar: "dagsterPlus" -sidebar_position: 2 +sidebar_position: 300 sidebar_label: "Set with agent config" unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/built-in.md b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/built-in.md index d81a2cbf93468..89eea22a2bef0 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/built-in.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/built-in.md @@ -1,7 +1,7 @@ --- title: "Built-in environment variables" displayed_sidebar: "dagsterPlus" -sidebar_position: 3 +sidebar_position: 100 sidebar_label: "Built-in variables" --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/dagster-ui.md b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/dagster-ui.md index 514a13fda0705..a4c76d4dba0b6 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/dagster-ui.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/dagster-ui.md @@ -1,7 +1,7 @@ --- title: "Setting environment variables with the Dagster+ UI" displayed_sidebar: "dagsterPlus" -sidebar_position: 1 +sidebar_position: 200 sidebar_label: "Set with Dagster+ UI" --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables.md b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/index.mdx similarity index 56% rename from docs/docs-beta/docs/dagster-plus/deployment/environment-variables.md rename to docs/docs-beta/docs/dagster-plus/deployment/environment-variables/index.mdx index 8c5b831ecc2d3..6184b57ef8d88 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/environment-variables.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/environment-variables/index.mdx @@ -2,6 +2,11 @@ title: "Environment variables" displayed_sidebar: "dagsterPlus" unlisted: true +sidebar_position: 40 --- # Environment variables + +import DocCardList from '@theme/DocCardList'; + + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-existing-vpc.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-existing-vpc.md index eab7a6cb68660..24ce5ad107359 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-existing-vpc.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-existing-vpc.md @@ -1,7 +1,7 @@ --- title: "Amazon ECS agents (Existing VPC)" displayed_sidebar: "dagsterPlus" -sidebar_position: 11 -sidebar_label: "Amazon ECS (existing)" +sidebar_position: 110 +sidebar_label: "Amazon ECS (Existing VPC)" unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-new-vpc.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-new-vpc.md index c2e4c4ab150e6..8cfbd7c7b48b0 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-new-vpc.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/amazon-ecs-new-vpc.md @@ -1,7 +1,7 @@ --- title: "Amazon ECS agents (New VPC)" displayed_sidebar: "dagsterPlus" -sidebar_position: 10 -sidebar_label: "Amazon ECS (new)" +sidebar_position: 100 +sidebar_label: "Amazon ECS (New VPC)" unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/docker.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/docker.md index 8ec342d1eda9f..ddd98a2f185a3 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/docker.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/docker.md @@ -1,7 +1,7 @@ --- title: "Docker agents" displayed_sidebar: "dagsterPlus" -sidebar_position: 20 +sidebar_position: 200 sidebar_label: "Docker" unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/index.mdx b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/index.mdx new file mode 100644 index 0000000000000..2bd0168cc8167 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/index.mdx @@ -0,0 +1,11 @@ +--- +title: "Agents" +displayed_sidebar: "dagsterPlus" +sidebar_class_name: hidden +--- + +# Agents + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/kubernetes.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/kubernetes.md index 5a8412a93da27..9169ef9783c29 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/kubernetes.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/kubernetes.md @@ -1,7 +1,7 @@ --- title: "Running Dagster+ agents on Kubernetes" displayed_sidebar: "dagsterPlus" -sidebar_position: 30 +sidebar_position: 300 sidebar_label: "Kubernetes" --- @@ -95,7 +95,7 @@ An exhaustive list of settings is available [here](/dagster-plus/deployment/hybr ### Configure your agents to serve branch deployments -[Branch deployments](/dagster-plus/deployment/branch-deployments) are lightweight staging environments created for each code change. To configure your Dagster+ agent to manage them: +[Branch deployments](/dagster-plus/features/branch-deployments/index.mdx) are lightweight staging environments created for each code change. To configure your Dagster+ agent to manage them: ```yaml # values.yaml diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/local.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/local.md index 5ae3fc34d5020..2c3c48aa94460 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/local.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/local.md @@ -1,7 +1,7 @@ --- title: "Running a Dagster+ agent locally" displayed_sidebar: "dagsterPlus" -sidebar_position: 40 +sidebar_position: 600 sidebar_label: "Local" --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/multiple.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/multiple.md index f41c22e684a55..0ec8c793882cb 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/multiple.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/multiple.md @@ -1,7 +1,7 @@ --- title: "Using multiple agents" displayed_sidebar: "dagsterPlus" -sidebar_position: 50 +sidebar_position: 400 unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/settings.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/settings.md index c086f9b11a154..5fa76858ea8f7 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/settings.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/agents/settings.md @@ -1,7 +1,7 @@ --- title: "Hybrid agent settings" displayed_sidebar: "dagsterPlus" -sidebar_position: 60 +sidebar_position: 700 sidebar_label: "Settings" unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/architecture.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/architecture.md index 5d012be9199e0..3209f88f7a55d 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/architecture.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/architecture.md @@ -1,7 +1,7 @@ --- title: 'Dagster+ Hybrid architecture' displayed_sidebar: 'dagsterPlus' -sidebar_position: 10 +sidebar_position: 100 --- # Dagster+ Hybrid architecture diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/index.md similarity index 94% rename from docs/docs-beta/docs/dagster-plus/deployment/hybrid.md rename to docs/docs-beta/docs/dagster-plus/deployment/hybrid/index.md index 631d802667a0c..5a7836ad79bcd 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/index.md @@ -1,7 +1,7 @@ --- title: "Hybrid deployment" displayed_sidebar: "dagsterPlus" -sidebar_position: 2 +sidebar_position: 20 --- # Hybrid deployment @@ -40,7 +40,7 @@ When you inform Dagster+ about a new code location, we enqueue instructions for ### Runs -Your definitions might include [automations](/guides/automation) that launch runs or materialize assets. Or your developers might launch runs directly with the web UI. +Your definitions might include [automations](/guides/automate) that launch runs or materialize assets. Or your developers might launch runs directly with the web UI. When a run needs to be launched, Dagster+ enqueues instructions for your agent to launch a new run. The next time your agent polls Dagster+ for new work, it will see instructions about how to launch your run. It will delegate those instructions to your code server and your code server will launch a run - a new run will typically require its own container. diff --git a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/tokens.md b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/tokens.md index ea705c4240bd8..133556526e526 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/hybrid/tokens.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/hybrid/tokens.md @@ -1,7 +1,7 @@ --- title: 'Hybrid agent tokens' displayed_sidebar: 'dagsterPlus' -sidebar_position: 30 +sidebar_position: 200 unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/migration/index.mdx b/docs/docs-beta/docs/dagster-plus/deployment/migration/index.mdx new file mode 100644 index 0000000000000..83ccdabe3b421 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/deployment/migration/index.mdx @@ -0,0 +1,11 @@ +--- +title: Migration +displayed_sidebar: "dagsterPlus" +sidebar_position: 30 +--- + +# Migration + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/self-hosted-to-dagster-plus.md b/docs/docs-beta/docs/dagster-plus/deployment/migration/self-hosted-to-dagster-plus.md similarity index 75% rename from docs/docs-beta/docs/guides/self-hosted-to-dagster-plus.md rename to docs/docs-beta/docs/dagster-plus/deployment/migration/self-hosted-to-dagster-plus.md index 715612374d783..daee67878b339 100644 --- a/docs/docs-beta/docs/guides/self-hosted-to-dagster-plus.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/migration/self-hosted-to-dagster-plus.md @@ -1,5 +1,5 @@ --- title: "Migrate from self-hosted to Dagster+" -sidebar_position: 70 +sidebar_position: 100 unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/multi-tenancy.md b/docs/docs-beta/docs/dagster-plus/deployment/multi-tenancy.md similarity index 76% rename from docs/docs-beta/docs/dagster-plus/multi-tenancy.md rename to docs/docs-beta/docs/dagster-plus/deployment/multi-tenancy.md index cc3192d1eebb4..28b41d6d7f9d3 100644 --- a/docs/docs-beta/docs/dagster-plus/multi-tenancy.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/multi-tenancy.md @@ -1,6 +1,7 @@ --- title: "Multi-tenancy" unlisted: true +sidebar_position: 500 --- # Dagster+ multi-tenancy diff --git a/docs/docs-beta/docs/dagster-plus/deployment/serverless/ci-cd-in-serverless.md b/docs/docs-beta/docs/dagster-plus/deployment/serverless/ci-cd-in-serverless.md index 36da2b6902e10..e27b93194a034 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/serverless/ci-cd-in-serverless.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/serverless/ci-cd-in-serverless.md @@ -1,17 +1,17 @@ --- title: 'CI/CD in Serverless ' displayed_sidebar: 'dagsterPlus' -sidebar_position: 10 +sidebar_position: 300 --- # CI/CD in Serverless -If you're a GitHub or GitLab user, you can use our predefined workflows to seamlessly deploy and synchronize your code to Dagster+. You can also use other Git providers or a local Git repository with our [dagster-cloud CLI](/dagster-plus/deployment/branch-deployments/dagster-cloud-cli) to run your own CI/CD process. +If you're a GitHub or GitLab user, you can use our predefined workflows to seamlessly deploy and synchronize your code to Dagster+. You can also use other Git providers or a local Git repository with our [dagster-cloud CLI](/dagster-plus/features/branch-deployments/dagster-cloud-cli) to run your own CI/CD process. -If you're a GitHub user, with a single click our GitHub app with GitHub Actions can set up a repo containing skeleton code and configuration for you consistent with Dagster+'s best practices. Pushing your code changes to the `main` branch will automatically deploy them to your `prod` Serverless deployment. Pull requests will spin up ephemeral [branch deployments](/dagster-plus/deployment/branch-deployments) that you can view in the Dagster+ UI for previewing and testing. +If you're a GitHub user, with a single click our GitHub app with GitHub Actions can set up a repo containing skeleton code and configuration for you consistent with Dagster+'s best practices. Pushing your code changes to the `main` branch will automatically deploy them to your `prod` Serverless deployment. Pull requests will spin up ephemeral [branch deployments](/dagster-plus/features/branch-deployments/index.mdx) that you can view in the Dagster+ UI for previewing and testing. :::note **If you are importing a Dagster project that's in an existing GitHub repo:** @@ -28,13 +28,13 @@ If you're a GitHub user, with a single click our GitHub app with GitHub Actions -If you're a GitLab user, with a single click our GitLab app can set up a repo containing skeleton code and CI/CD configuration for you consistent with Dagster+'s best practices. Pushing your code changes to the `main` branch will automatically deploy them to your `prod` Serverless deployment. Pull requests will spin up ephemeral [branch deployments](/dagster-plus/deployment/branch-deployments) that you can view in the Dagster+ UI for previewing and testing. +If you're a GitLab user, with a single click our GitLab app can set up a repo containing skeleton code and CI/CD configuration for you consistent with Dagster+'s best practices. Pushing your code changes to the `main` branch will automatically deploy them to your `prod` Serverless deployment. Pull requests will spin up ephemeral [branch deployments](/dagster-plus/features/branch-deployments/index.mdx) that you can view in the Dagster+ UI for previewing and testing. -If you don't want to use our automated GitHub/GitLab process, we offer [the powerful `dagster-cloud` command-line interface (CLI)](/dagster-plus/deployment/branch-deployments/dagster-cloud-cli) that you can use in another CI environment or locally. +If you don't want to use our automated GitHub/GitLab process, we offer [the powerful `dagster-cloud` command-line interface (CLI)](/dagster-plus/features/branch-deployments/dagster-cloud-cli) that you can use in another CI environment or locally. First, [create a new project](/getting-started/quickstart) with the Dagster open source CLI. @@ -62,7 +62,7 @@ install_requires=[ ::: -Next, install the [`dagster-cloud` CLI](/dagster-plus/deployment/branch-deployments/dagster-cloud-cli) and use its `configure` command to authenticate it to your Dagster+ organization. +Next, install the [`dagster-cloud` CLI](/dagster-plus/features/branch-deployments/dagster-cloud-cli) and use its `configure` command to authenticate it to your Dagster+ organization. **Note:** The CLI requires a recent version of Python 3 and Docker. @@ -71,7 +71,7 @@ pip install dagster-cloud dagster-cloud configure ``` -You can also configure the `dagster-cloud` tool non-interactively; see [the CLI docs](/dagster-plus/deployment/branch-deployments/dagster-cloud-cli) for more information. +You can also configure the `dagster-cloud` tool non-interactively; see [the CLI docs](/dagster-plus/features/branch-deployments/dagster-cloud-cli) for more information. Finally, deploy your project to Dagster+ using the `serverless` command: diff --git a/docs/docs-beta/docs/dagster-plus/deployment/serverless/dagster-ips.md b/docs/docs-beta/docs/dagster-plus/deployment/serverless/dagster-ips.md index 8c517908c5cdb..9b15bb595293e 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/serverless/dagster-ips.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/serverless/dagster-ips.md @@ -1,7 +1,7 @@ --- title: 'IP addresses' displayed_sidebar: 'dagsterPlus' -sidebar_position: 50 +sidebar_position: 400 --- # Dagster+ Serverless IP addresses diff --git a/docs/docs-beta/docs/dagster-plus/deployment/serverless.md b/docs/docs-beta/docs/dagster-plus/deployment/serverless/index.md similarity index 98% rename from docs/docs-beta/docs/dagster-plus/deployment/serverless.md rename to docs/docs-beta/docs/dagster-plus/deployment/serverless/index.md index 898b657c26a55..0684a378be624 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/serverless.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/serverless/index.md @@ -1,7 +1,7 @@ --- title: 'Serverless deployment' displayed_sidebar: 'dagsterPlus' -sidebar_position: 1 +sidebar_position: 10 --- # Serverless deployment diff --git a/docs/docs-beta/docs/dagster-plus/deployment/serverless/run-isolation.md b/docs/docs-beta/docs/dagster-plus/deployment/serverless/run-isolation.md index 8efe4af99ff27..68b58ee1231b5 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/serverless/run-isolation.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/serverless/run-isolation.md @@ -2,7 +2,7 @@ title: 'Serverless run isolation' displayed_sidebar: 'dagsterPlus' sidebar_label: 'Run isolation' -sidebar_position: 35 +sidebar_position: 200 --- # Serverless run isolation @@ -15,7 +15,7 @@ Dagster+ Serverless offers two settings for run isolation: isolated and non-isol To follow the steps in this guide, you'll need: - An active Dagster+ Serverless deployment -- An understanding of [Dagster+ deployment settings](/dagster-plus/deployment/settings) +- An understanding of [Dagster+ deployment settings](/dagster-plus/deployment/deployment-settings) --- @@ -57,7 +57,7 @@ While launching runs from the Launchpad, un-check `Isolate run environment`. Whe To avoid crashes and memory exhaustion, only one non-isolated run will execute at once by default. While a non-isolated run is in progress, the Launchpad will only allow isolated runs to be launched. -This limit can be configured in [deployment settings](/dagster-plus/deployment/settings). +This limit can be configured in [deployment settings](/dagster-plus/deployment/deployment-settings). ```yaml non_isolated_runs: diff --git a/docs/docs-beta/docs/dagster-plus/deployment/serverless/runtime-environment.md b/docs/docs-beta/docs/dagster-plus/deployment/serverless/runtime-environment.md index a1848dfe83877..76cf15cb741db 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/serverless/runtime-environment.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/serverless/runtime-environment.md @@ -2,7 +2,7 @@ title: 'Serverless runtime environment' displayed_sidebar: 'dagsterPlus' sidebar_label: 'Runtime environment' -sidebar_position: 30 +sidebar_position: 100 --- By default, Dagster+ Serverless will package your code as PEX files and deploys them on Docker images. Using PEX files significantly reduces the time to deploy since it does not require building a new Docker image and provisioning a new container for every code change. However you are able to customize the Serverless runtime environment in various ways: @@ -203,7 +203,7 @@ Setting a custom base image isn't supported for GitLab CI/CD workflows out of th ## Use private Python packages \{#private-packages} -If you use PEX deploys in your workflow (`ENABLE_FAST_DEPLOYS: 'true'`), the following steps can install a package from a private GitHub repository, eg. `my-org/private-repo`, as a dependency: +If you use PEX deploys in your workflow (`ENABLE_FAST_DEPLOYS: 'true'`), the following steps can install a package from a private GitHub repository, e.g. `my-org/private-repo`, as a dependency: 1. In your `deploy.yml` file, add the following to the top of `steps:` section in the `dagster-cloud-default-deploy` job. diff --git a/docs/docs-beta/docs/dagster-plus/deployment/serverless/security.md b/docs/docs-beta/docs/dagster-plus/deployment/serverless/security.md index 09f33964ac36e..d6a29601a7f44 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/serverless/security.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/serverless/security.md @@ -1,7 +1,7 @@ --- title: 'Security & data protection' displayed_sidebar: 'dagsterPlus' -sidebar_position: 20 +sidebar_position: 500 --- # Serverless security & data protection @@ -24,11 +24,11 @@ The default I/O manager cannot be used if you are a Serverless user who: - Are otherwise working with data subject to GDPR or other such regulations ::: -In Serverless, code that uses the default [I/O manager](/guides/io-managers) is automatically adjusted to save data in Dagster+ managed storage. This automatic change is useful because the Serverless filesystem is ephemeral, which means the default I/O manager wouldn't work as expected. +In Serverless, code that uses the default [I/O manager](/guides/build/configure/io-managers) is automatically adjusted to save data in Dagster+ managed storage. This automatic change is useful because the Serverless filesystem is ephemeral, which means the default I/O manager wouldn't work as expected. However, this automatic change also means potentially sensitive data could be **stored** and not just processed or orchestrated by Dagster+. -To prevent this, you can use [another I/O manager](/guides/io-managers#built-in) that stores data in your infrastructure or [adapt your code to avoid using an I/O manager](/guides/io-managers#before-you-begin). +To prevent this, you can use [another I/O manager](/guides/build/configure/io-managers#built-in) that stores data in your infrastructure or [adapt your code to avoid using an I/O manager](/guides/build/configure/io-managers#before-you-begin). :::note You must have [boto3](https://pypi.org/project/boto3/) or `dagster-cloud[serverless]` installed as a project dependency otherwise the Dagster+ managed storage can fail and silently fall back to using the default I/O manager. diff --git a/docs/docs-beta/docs/dagster-plus/deployment/serverless/transition-hybrid.md b/docs/docs-beta/docs/dagster-plus/deployment/serverless/transition-hybrid.md index 6790e20c2cbf5..1ce8719fbde5e 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/serverless/transition-hybrid.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/serverless/transition-hybrid.md @@ -1,7 +1,7 @@ --- title: "Transitioning to Hybrid" displayed_sidebar: "dagsterPlus" -sidebar_position: 50 +sidebar_position: 600 --- After utilizing a Dagster+ [Serverless](/dagster-plus/deployment/serverless) deployment, you may decide to leverage your own infrastructure to execute your code. Transitioning to a Hybrid deployment requires only a few steps and can be done without any loss of execution history or metadata, allowing you to maintain continuity and control over your operations. diff --git a/docs/docs-beta/docs/dagster-plus/deployment/tokens.md b/docs/docs-beta/docs/dagster-plus/deployment/user-agent-tokens.md similarity index 84% rename from docs/docs-beta/docs/dagster-plus/deployment/tokens.md rename to docs/docs-beta/docs/dagster-plus/deployment/user-agent-tokens.md index 62a79168b2352..8a1b94d891046 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/tokens.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/user-agent-tokens.md @@ -2,6 +2,7 @@ title: "Tokens" displayed_sidebar: "dagsterPlus" unlisted: true +sidebar_position: 600 --- # Managing user and agent tokens in Dagster+ diff --git a/docs/docs-beta/docs/dagster-plus/deployment/alerts/cli.md b/docs/docs-beta/docs/dagster-plus/features/alerts/cli.md similarity index 79% rename from docs/docs-beta/docs/dagster-plus/deployment/alerts/cli.md rename to docs/docs-beta/docs/dagster-plus/features/alerts/cli.md index 9875be88b9b7c..71e28ce328c5e 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/alerts/cli.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/cli.md @@ -1,6 +1,7 @@ --- title: "Dagster+ alerts with the CLI" unlisted: true +sidebar_position: 200 --- # Alerts with the CLI diff --git a/docs/docs-beta/docs/dagster-plus/deployment/alerts/email.md b/docs/docs-beta/docs/dagster-plus/features/alerts/email.md similarity index 78% rename from docs/docs-beta/docs/dagster-plus/deployment/alerts/email.md rename to docs/docs-beta/docs/dagster-plus/features/alerts/email.md index b99b231da11ac..2b516c6b73bac 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/alerts/email.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/email.md @@ -1,6 +1,7 @@ --- title: "Dagster+ email alerts" unlisted: true +sidebar_position: 300 --- # Dagster+ email alerts diff --git a/docs/docs-beta/docs/dagster-plus/deployment/alerts.md b/docs/docs-beta/docs/dagster-plus/features/alerts/index.md similarity index 100% rename from docs/docs-beta/docs/dagster-plus/deployment/alerts.md rename to docs/docs-beta/docs/dagster-plus/features/alerts/index.md index 37c7b6c897035..b869009c354d0 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/alerts.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/index.md @@ -1,7 +1,7 @@ --- title: Setting up alerts on Dagster+ -sidebar_position: 30 sidebar_label: "Dagster+ Alerts" +sidebar_position: 30 --- [comment]: <> (This file is automatically generated by `dagster-plus/deployment/alerts/generate_alerts_doc.py`) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/alerts/microsoft-teams.md b/docs/docs-beta/docs/dagster-plus/features/alerts/microsoft-teams.md similarity index 64% rename from docs/docs-beta/docs/dagster-plus/deployment/alerts/microsoft-teams.md rename to docs/docs-beta/docs/dagster-plus/features/alerts/microsoft-teams.md index 87dcdbecf01a8..7c66b3e2dae8b 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/alerts/microsoft-teams.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/microsoft-teams.md @@ -1,6 +1,8 @@ --- title: "Dagster+ Microsoft Teams alerts" unlisted: true +sidebar_label: "Microsoft Teams" +sidebar_position: 400 --- # Dagster+ Microsoft Teams alerts diff --git a/docs/docs-beta/docs/dagster-plus/deployment/alerts/pagerduty.md b/docs/docs-beta/docs/dagster-plus/features/alerts/pagerduty.md similarity index 63% rename from docs/docs-beta/docs/dagster-plus/deployment/alerts/pagerduty.md rename to docs/docs-beta/docs/dagster-plus/features/alerts/pagerduty.md index 6348eb84d7608..c8a32645e592f 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/alerts/pagerduty.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/pagerduty.md @@ -1,6 +1,8 @@ --- title: "Dagster+ PagerDuty alerts" unlisted: true +sidebar_label: "PagerDuty" +sidebar_position: 500 --- # Dagster+ PagerDuty alerts diff --git a/docs/docs-beta/docs/dagster-plus/deployment/alerts/slack.md b/docs/docs-beta/docs/dagster-plus/features/alerts/slack.md similarity index 63% rename from docs/docs-beta/docs/dagster-plus/deployment/alerts/slack.md rename to docs/docs-beta/docs/dagster-plus/features/alerts/slack.md index 7750f783e4e1f..3d9f5423dd890 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/alerts/slack.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/slack.md @@ -1,6 +1,8 @@ --- title: "Dagster+ Slack alerts" unlisted: true +sidebar_label: "Slack" +sidebar_position: 600 --- # Dagster+ Slack alerts diff --git a/docs/docs-beta/docs/dagster-plus/deployment/alerts/ui.md b/docs/docs-beta/docs/dagster-plus/features/alerts/ui.md similarity index 65% rename from docs/docs-beta/docs/dagster-plus/deployment/alerts/ui.md rename to docs/docs-beta/docs/dagster-plus/features/alerts/ui.md index 15555e841c632..b613282af846e 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/alerts/ui.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/ui.md @@ -1,6 +1,8 @@ --- title: "Dagster+ alerts in the UI" unlisted: true +sidebar_label: "UI" +sidebar_position: 100 --- # Alerts in the UI diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/index.mdx b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/index.mdx new file mode 100644 index 0000000000000..a17a4992483d4 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/index.mdx @@ -0,0 +1,11 @@ +--- +title: "Authentication and access control" +displayed_sidebar: "dagsterPlus" +sidebar_position: 40 +--- + +# Authentication and access control + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/dagster-plus/access/rbac/audit-logs.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/audit-logs.md similarity index 87% rename from docs/docs-beta/docs/dagster-plus/access/rbac/audit-logs.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/audit-logs.md index fc574334a960a..cfc6c1cd81337 100644 --- a/docs/docs-beta/docs/dagster-plus/access/rbac/audit-logs.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/audit-logs.md @@ -1,7 +1,7 @@ --- title: "Audit logs" displayed_sidebar: "dagsterPlus" -sidebar_position: 4 +sidebar_position: 300 --- # Audit logs @@ -16,7 +16,7 @@ This guide walks through how to access the audit log and details the interaction
Prerequisites - A Dagster+ Pro organization -- An [Organization Admin](/dagster-plus/access/rbac/user-roles-permissions) role in your Dagster+ organization +- An [Organization Admin](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) role in your Dagster+ organization
## View audit logs @@ -46,15 +46,15 @@ The **Filter** button near the top left of the page can be used to filter the li | Log in | A user logs in to the Dagster+ organization | | | Update sensor | A user toggles a sensor on or off | The sensor name, code location, and cursor | | Update schedule | A user toggles a schedule on or off | The schedule name, code location, and cursor | -| Update alert policy | A user modifies an [alert policy](/dagster-plus/deployment/alerts/ui) | The new configuration for the alert policy | +| Update alert policy | A user modifies an [alert policy](/dagster-plus/features/alerts/ui) | The new configuration for the alert policy | | Create deployment | A user creates a new deployment | Whether the deployment is a branch deployment | | Delete deployment | A user removes an existing deployment | Whether the deployment is a branch deployment | | Create user token | A user creates a new user token | | | Revoke user token | A user revokes an existing user token | | -| Change user permissions | A user alters [permissions](/dagster-plus/access/rbac/user-roles-permissions) for another user | The permission grant and targeted deployment | +| Change user permissions | A user alters [permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) for another user | The permission grant and targeted deployment | | Create agent token | A user creates a new agent token | | | Revoke agent token | A user revokes an existing agent token | | -| Update agent token permissions | A user alters [permissions](/dagster-plus/access/rbac/user-roles-permissions) for an agent token | The permission grant and targeted deployment | +| Update agent token permissions | A user alters [permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) for an agent token | The permission grant and targeted deployment | | Create secret | A user creates a new [environment variable](/dagster-plus/deployment/environment-variables/dagster-ui) | The created variable name | | Update secret | A user modifies an existing [environment variable](/dagster-plus/deployment/environment-variables/dagster-ui) | The previous and current variable names and whether the value was changed | | Delete secret | A user removes an [environment variable](/dagster-plus/deployment/environment-variables/dagster-ui) | The deleted variable name | diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/index.mdx b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/index.mdx new file mode 100644 index 0000000000000..1443bd749b3b4 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/index.mdx @@ -0,0 +1,11 @@ +--- +title: "Role-based access control" +displayed_sidebar: "dagsterPlus" +sidebar_position: 10 +--- + +# Role-based access control + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/dagster-plus/access/rbac/teams.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/teams.md similarity index 73% rename from docs/docs-beta/docs/dagster-plus/access/rbac/teams.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/teams.md index e45bb82c261a1..dff5866950069 100644 --- a/docs/docs-beta/docs/dagster-plus/access/rbac/teams.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/teams.md @@ -1,12 +1,12 @@ --- title: "Team management" displayed_sidebar: "dagsterPlus" -sidebar_position: 2 +sidebar_position: 100 --- # Team management in Dagster+ -As part of [role-based access control (RBAC)](/dagster-plus/access/rbac/user-roles-permissions), Dagster+ supports the ability to assign users to teams. A team is a group of users with a set of default deployment, code location, and Branch Deployment user roles. +As part of [role-based access control (RBAC)](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions), Dagster+ supports the ability to assign users to teams. A team is a group of users with a set of default deployment, code location, and Branch Deployment user roles.
Prerequisites @@ -14,7 +14,7 @@ As part of [role-based access control (RBAC)](/dagster-plus/access/rbac/user-rol To complete the steps in this guide, you'll need: - A Dagster+ Pro plan -- Dagster+ [Organization Admin permissions](/dagster-plus/access/rbac/user-roles-permissions): +- Dagster+ [Organization Admin permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions): - In your organization, and - For the deployments where you want to manage teams @@ -56,9 +56,9 @@ Navigate to the **Organization Settings > Teams** tab and locate the team you wa 1. Click the **Edit** button in the **Actions** column. 2. In the **Roles** tab, click the **Edit team role** button next to the deployment where you want to modify the team's role. -3. In the window that displays, select the team role for the deployment. This [role](/dagster-plus/access/rbac/user-roles-permissions) will be used as the default for this team for all code locations in the deployment. +3. In the window that displays, select the team role for the deployment. This [role](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) will be used as the default for this team for all code locations in the deployment. 4. Click **Save**. -5. To set permissions for individual [code locations](/dagster-plus/access/rbac/user-roles-permissions) in a deployment: +5. To set permissions for individual [code locations](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) in a deployment: 1. Click the toggle to the left of the deployment to open a list of code locations. 2. Next to a code location, click **Edit team role**. 3. Select the team role for the code location. @@ -74,5 +74,5 @@ Navigate to the **Organization Settings > Teams** tab and locate the team you wa ## Next steps -- Learn more about RBAC in [Understanding User Roles & Permissions](/dagster-plus/access/rbac/user-roles-permissions) -- Learn more about how to manage users in Dagster+ in [Understanding User Management in Dagster+](/dagster-plus/access/rbac/users) +- Learn more about RBAC in [Understanding User Roles & Permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) +- Learn more about how to manage users in Dagster+ in [Understanding User Management in Dagster+](/dagster-plus/features/authentication-and-access-control/rbac/users) diff --git a/docs/docs-beta/docs/dagster-plus/access/rbac/user-roles-permissions.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions.md similarity index 86% rename from docs/docs-beta/docs/dagster-plus/access/rbac/user-roles-permissions.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions.md index c8e94ca526d51..6281ae236cd9e 100644 --- a/docs/docs-beta/docs/dagster-plus/access/rbac/user-roles-permissions.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions.md @@ -1,7 +1,7 @@ --- title: 'User roles & permissions' displayed_sidebar: 'dagsterPlus' -sidebar_position: 3 +sidebar_position: 200 --- # Understanding user roles & permissions in Dagster+ @@ -38,13 +38,13 @@ All user roles are enforced both in Dagster+ and the GraphQL API. ### Teams -Dagster+ Pro users can create teams of users and assign default permission sets. Refer to the [Managing teams in Dagster+](/dagster-plus/access/rbac/teams) guide for more info. +Dagster+ Pro users can create teams of users and assign default permission sets. Refer to the [Managing teams in Dagster+](/dagster-plus/features/authentication-and-access-control/rbac/teams) guide for more info. ## Assigning user and team roles With the exception of the **Organization Admin** role, user and team roles are set on a per-deployment basis. -Organization Admins have access to the entire organization, including all [deployments](/todo), [code locations](/dagster-plus/deployment/code-locations), and [Branch Deployments](/dagster-plus/deployment/branch-deployments). +Organization Admins have access to the entire organization, including all [deployments](/todo), [code locations](/dagster-plus/deployment/code-locations), and [Branch Deployments](dagster-plus/features/branch-deployments/index.mdx). | Level | Plan | Description | | ------------------ | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -74,7 +74,7 @@ To override a code location role for an individual user: #### Team members -Users in your organization can belong to one or more [teams](/dagster-plus/access/rbac/teams). When determining a user's level of access, Dagster+ will use the **most permissive** role assigned to the user between all of their team memberships and any individual role grants. +Users in your organization can belong to one or more [teams](/dagster-plus/features/authentication-and-access-control/rbac/teams). When determining a user's level of access, Dagster+ will use the **most permissive** role assigned to the user between all of their team memberships and any individual role grants. For example, let's look at a user with the following roles for our `dev` deployment: @@ -112,12 +112,12 @@ TODO: add picture previously at "/images/dagster-cloud/user-token-management/cod | | Viewer | Launcher | Editor | Admin | Organization
admin | | ------------------------------------------------------------------------ | ------ | -------- | ------ | ----- | ------------------------ | -| View runs of [jobs](/concepts/ops-jobs) | ✅ | ✅ | ✅ | ✅ | ✅ | +| View runs of [jobs](/guides/build/ops-jobs) | ✅ | ✅ | ✅ | ✅ | ✅ | | Launch, re-execute, terminate, and delete runs of jobs | ❌ | ✅ | ✅ | ✅ | ✅ | -| Start and stop [schedules](/concepts/schedules) | ❌ | ❌ | ✅ | ✅ | ✅ | -| Start and stop [schedules](/concepts/sensors) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Start and stop [schedules](/guides/automate/schedules) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Start and stop [schedules](/guides/automate/sensors) | ❌ | ❌ | ✅ | ✅ | ✅ | | Wipe assets | ❌ | ❌ | ✅ | ✅ | ✅ | -| Launch and cancel [schedules](/guides/backfill) | ❌ | ✅ | ✅ | ✅ | ✅ | +| Launch and cancel [schedules](/guides/build/backfill) | ❌ | ✅ | ✅ | ✅ | ✅ | | Add dynamic partitions | ❌ | ❌ | ✅ | ✅ | ✅ | ### Deployments @@ -132,7 +132,7 @@ Deployment settings are accessed in the UI by navigating to **user menu (your ic | View [environment variable](/dagster-plus/deployment/environment-variables) values | ❌ | ❌ | ✅ | ✅ | ✅ | | Export [environment variables](/dagster-plus/deployment/environment-variables) | ❌ | ❌ | ✅ | ✅ | ✅ | | Create and delete [deployments](/todo) | ❌ | ❌ | ❌ | ❌ | ✅ | -| Create [Branch Deployments](/dagster-plus/deployment/branch-deployments) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Create [Branch Deployments](dagster-plus/features/branch-deployments/index.mdx) | ❌ | ❌ | ✅ | ✅ | ✅ | ### Code locations @@ -171,7 +171,7 @@ User management is accessed in the UI by navigating to **user menu (your icon) > | | Viewer | Launcher | Editor | Admin | Organization
admin | | --------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | -| [View users](/dagster-plus/access/rbac/users) | ✅ | ✅ | ✅ | ✅ | ✅ | +| [View users](/dagster-plus/features/authentication-and-access-control/rbac/users) | ✅ | ✅ | ✅ | ✅ | ✅ | | Add users | ❌ | ❌ | ❌ | ✅ | ✅ | | Edit user roles | ❌ | ❌ | ❌ | ❌ | ✅ | | Remove users | ❌ | ❌ | ❌ | ❌ | ✅ | @@ -184,7 +184,7 @@ Team management is accessed in the UI by navigating to **user menu (your icon) > | | Viewer | Launcher | Editor | Admin | Organization
admin | | --------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | -| [View teams](/dagster-plus/access/rbac/teams) | ✅ | ✅ | ✅ | ✅ | ✅ | +| [View teams](/dagster-plus/features/authentication-and-access-control/rbac/teams) | ✅ | ✅ | ✅ | ✅ | ✅ | | Modify team permissions | ❌ | ❌ | ❌ | ✅ | ✅ | | Create teams | ❌ | ❌ | ❌ | ❌ | ✅ | | Re-name teams | ❌ | ❌ | ❌ | ❌ | ✅ | @@ -195,9 +195,9 @@ Team management is accessed in the UI by navigating to **user menu (your icon) > | | Viewer | Launcher | Editor | Admin | Organization
admin | | ------------------------------------------------------ | ------ | -------- | ------ | ----- | ------------------------ | -| Manage [alerts](/dagster-plus/deployment/alerts) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Manage [alerts](/dagster-plus/features/alerts) | ❌ | ❌ | ✅ | ✅ | ✅ | | Edit workspace | ❌ | ❌ | ✅ | ✅ | ✅ | -| [Administer SAML](/dagster-plus/access/authentication) | ❌ | ❌ | ❌ | ❌ | ✅ | +| [Administer SAML](/dagster-plus/features/authentication-and-access-control/sso/authentication) | ❌ | ❌ | ❌ | ❌ | ✅ | | [Manage SCIM](/todo) | ❌ | ❌ | ❌ | ❌ | ✅ | | View usage | ❌ | ❌ | ❌ | ❌ | ✅ | | Manage billing | ❌ | ❌ | ❌ | ❌ | ✅ | @@ -205,7 +205,7 @@ Team management is accessed in the UI by navigating to **user menu (your icon) > ## Next steps -- Learn more about how to manage users in Dagster+ in [Understanding User Management in Dagster+](/dagster-plus/access/rbac/users) -- Learn more about how to manage teams in Dagster+ in [Understanding Team Management in Dagster+](/dagster-plus/access/rbac/teams) -- Learn more about SCIM provisioning in [Understanding SCIM Provisioning](/todo) -- Learn more about authentication in [Understanding Authentication](/dagster-plus/access/authentication) +- Learn more about how to manage users in Dagster+ in [Understanding User Management in Dagster+](/dagster-plus/features/authentication-and-access-control/rbac/users) +- Learn more about how to manage teams in Dagster+ in [Understanding Team Management in Dagster+](/dagster-plus/features/authentication-and-access-control/rbac/teams) +- Learn more about SCIM provisioning in [Understanding SCIM Provisioning](/dagster-plus/features/authentication-and-access-control/scim/index.mdx) +- Learn more about authentication in [Understanding Authentication](/dagster-plus/features/authentication-and-access-control/sso/authentication) diff --git a/docs/docs-beta/docs/dagster-plus/access/rbac/users.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/users.md similarity index 90% rename from docs/docs-beta/docs/dagster-plus/access/rbac/users.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/users.md index 0fa920b4c4ca9..61c3a2f55401f 100644 --- a/docs/docs-beta/docs/dagster-plus/access/rbac/users.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/users.md @@ -2,7 +2,7 @@ title: "Managing users in Dagster+" displayed_sidebar: "dagsterPlus" sidebar_label: "User management" -sidebar_position: 10 +sidebar_position: 400 --- Dagster+ allows you to grant specific permissions to your organization's users, ensuring that Dagster users have access only to what they require. @@ -87,7 +87,7 @@ Removing a user removes them from the organization. **Note**: If using a SAML-ba ## Next steps -- Learn more about role-based access control (RBAC) in [Understanding User Roles & Permissions](/dagster-plus/access/rbac/user-roles-permissions) -- Learn more about how to manage teams in Dagster+ in [Understanding Team Management in Dagster+](/dagster-plus/access/rbac/teams) -- Learn more about SCIM provisioning in [Understanding SCIM Provisioning](/dagster-plus/access/authentication/scim-provisioning) -- Learn more about authentication in [Understanding Authentication](/dagster-plus/access/authentication) +- Learn more about role-based access control (RBAC) in [Understanding User Roles & Permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) +- Learn more about how to manage teams in Dagster+ in [Understanding Team Management in Dagster+](/dagster-plus/features/authentication-and-access-control/rbac/teams) +- Learn more about SCIM provisioning in [SCIM Provisioning](/dagster-plus/features/authentication-and-access-control/scim) +- Learn more about authentication in [Understanding Authentication](/dagster-plus/features/authentication-and-access-control/sso/authentication) diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/azure-active-directory.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/azure-active-directory.md new file mode 100644 index 0000000000000..28fd1473a5e56 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/azure-active-directory.md @@ -0,0 +1,10 @@ +--- +title: 'Azure Active Directory' +displayed_sidebar: 'dagsterPlus' +unlisted: true +sidebar_position: 200 +--- + +# Azure Active Directory + +See the [Dagster Cloud provisioning tutorial](https://learn.microsoft.com/en-us/azure/active-directory/saas-apps/dagster-cloud-provisioning-tutorial). diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/index.mdx b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/index.mdx new file mode 100644 index 0000000000000..19ee9983dc77d --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/index.mdx @@ -0,0 +1,11 @@ +--- +title: "SCIM provisioning" +displayed_sidebar: "dagsterPlus" +sidebar_position: 30 +--- + +# SCIM provisioning + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication/okta-scim.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/okta-scim.md similarity index 83% rename from docs/docs-beta/docs/dagster-plus/access/authentication/okta-scim.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/okta-scim.md index d7ecf4beb9c22..a1dd29b1bf275 100644 --- a/docs/docs-beta/docs/dagster-plus/access/authentication/okta-scim.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/okta-scim.md @@ -2,6 +2,7 @@ title: 'Okta SCIM provisioning' displayed_sidebar: 'dagsterPlus' unlisted: true +sidebar_position: 100 --- # Okta SCIM provisioning diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/authentication.md similarity index 81% rename from docs/docs-beta/docs/dagster-plus/access/authentication.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/authentication.md index a97dee5f69a1c..2c626adac16d8 100644 --- a/docs/docs-beta/docs/dagster-plus/access/authentication.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/authentication.md @@ -2,6 +2,7 @@ title: "Authentication" displayed_sidebar: "dagsterPlus" unlisted: true +sidebar_position: 100 --- # Authentication diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication/azure-ad-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/azure-ad-sso.md similarity index 94% rename from docs/docs-beta/docs/dagster-plus/access/authentication/azure-ad-sso.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/azure-ad-sso.md index 316f793c67ca0..5921d30d4d995 100644 --- a/docs/docs-beta/docs/dagster-plus/access/authentication/azure-ad-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/azure-ad-sso.md @@ -1,6 +1,7 @@ --- title: 'Azure Active Directory SSO' displayed_sidebar: 'dagsterPlus' +sidebar_position: 200 --- # Setting up Azure Active Directory SSO for Dagster+ @@ -17,7 +18,7 @@ To complete the steps in this guide, you'll need: - **The following in Dagster+:** - A Pro plan - [Access to a user token](/todo) - - [Organization Admin permissions](/dagster-plus/access/rbac/user-roles-permissions) in your organization + - [Organization Admin permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) in your organization
@@ -88,7 +89,7 @@ In this section, you'll create a test user in the Azure portal. - Select the **Show password** checkbox and write down the value displayed in the **Password** box. 5. Click **Create**. -import TestSSO from '../../../partials/\_TestSSO.md'; +import TestSSO from '../../../../partials/\_TestSSO.md'; diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication/google-workspace-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/google-workspace-sso.md similarity index 94% rename from docs/docs-beta/docs/dagster-plus/access/authentication/google-workspace-sso.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/google-workspace-sso.md index d9a019c4c5d91..49727f24ad5c2 100644 --- a/docs/docs-beta/docs/dagster-plus/access/authentication/google-workspace-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/google-workspace-sso.md @@ -1,6 +1,7 @@ --- title: 'Google Workspace SSO' displayed_sidebar: 'dagsterPlus' +sidebar_position: 300 --- # Setting up Google Workspace SSO for Dagster+ @@ -19,7 +20,7 @@ To complete the steps in this guide, you'll need: - **The following in Dagster+:** - A Pro plan - [Access to a user token](/todo) - - [Organization Admin permissions](/dagster-plus/access/rbac/user-roles-permissions) in your organization + - [Organization Admin permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) in your organization @@ -103,7 +104,7 @@ In this step, you'll assign users in your Google Workspace to the Dagster+ appli ![Assign New Login](/img/placeholder.svg) -import TestSSO from '../../../partials/\_TestSSO.md'; +import TestSSO from '../../../../partials/\_TestSSO.md'; diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/index.mdx b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/index.mdx new file mode 100644 index 0000000000000..fe5219e36a95f --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/index.mdx @@ -0,0 +1,11 @@ +--- +title: "Single sign-on" +displayed_sidebar: "dagsterPlus" +sidebar_position: 20 +--- + +# Single sign-on + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication/okta-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/okta-sso.md similarity index 93% rename from docs/docs-beta/docs/dagster-plus/access/authentication/okta-sso.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/okta-sso.md index 1fadb6e5c4c5d..2b9d01f0aeda1 100644 --- a/docs/docs-beta/docs/dagster-plus/access/authentication/okta-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/okta-sso.md @@ -1,6 +1,7 @@ --- title: 'Okta SSO' displayed_sidebar: 'dagsterPlus' +sidebar_position: 400 --- # Setting up Okta SSO for Dagster+ @@ -17,7 +18,7 @@ To complete the steps in this guide, you'll need: - **The following in Dagster+:** - A Pro plan - [Access to a user token](/todo) - - [Organization Admin permissions](/dagster-plus/access/rbac/user-roles-permissions) in your organization + - [Organization Admin permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) in your organization @@ -83,7 +84,7 @@ Next, you'll assign users to the Dagster+ application in Okta. This will allow t 2. Click **Assign > Assign to People**. 3. For each user you want to have access to Dagster+, click **Assign** then **Save and Go Back**. -import TestSSO from '../../../partials/\_TestSSO.md'; +import TestSSO from '../../../../partials/\_TestSSO.md'; diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication/onelogin-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/onelogin-sso.md similarity index 92% rename from docs/docs-beta/docs/dagster-plus/access/authentication/onelogin-sso.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/onelogin-sso.md index d15f9576acb62..4b9c10a57c1a7 100644 --- a/docs/docs-beta/docs/dagster-plus/access/authentication/onelogin-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/onelogin-sso.md @@ -1,6 +1,7 @@ --- title: 'OneLogin SSO' displayed_sidebar: 'dagsterPlus' +sidebar_position: 500 --- # OneLogin SSO @@ -20,7 +21,7 @@ To follow the steps in this guide, you'll need: - **The following in Dagster+:** - A Pro plan - [Access to a user token](/todo) - - [Organization Admin permissions](/dagster-plus/access/rbac/user-roles-permissions) in your organization + - [Organization Admin permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) in your organization @@ -80,7 +81,7 @@ Next, you'll assign users to the Dagster+ application in OneLogin. This will all 7. Repeat steps 2-6 for every user you want to access Dagster+. -import TestSSO from '../../../partials/\_TestSSO.md'; +import TestSSO from '../../../../partials/\_TestSSO.md'; diff --git a/docs/docs-beta/docs/dagster-plus/access/authentication/pingone-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/pingone-sso.md similarity index 94% rename from docs/docs-beta/docs/dagster-plus/access/authentication/pingone-sso.md rename to docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/pingone-sso.md index c4ef93b0d0ee4..2c766cb1cfc70 100644 --- a/docs/docs-beta/docs/dagster-plus/access/authentication/pingone-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/pingone-sso.md @@ -1,6 +1,7 @@ --- title: 'PingOne SSO' displayed_sidebar: 'dagsterPlus' +sidebar_position: 600 --- # Setting up PingOne SSO for Dagster+ @@ -19,7 +20,7 @@ To complete the steps in this guide, you'll need: - **The following in Dagster+:** - A Pro plan - [Access to a user token](/todo) - - [Organization Admin permissions](/dagster-plus/access/rbac/user-roles-permissions) in your organization + - [Organization Admin permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) in your organization @@ -109,7 +110,7 @@ Next, you'll assign users to the Dagster+ application in PingOne. This will allo 3. Edit the policy as needed to grant users access to the application. -import TestSSO from '../../../partials/\_TestSSO.md'; +import TestSSO from '../../../../partials/\_TestSSO.md'; diff --git a/docs/docs-beta/docs/dagster-plus/features/branch-deployments/change-tracking.md b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/change-tracking.md new file mode 100644 index 0000000000000..40b5a837ae8d6 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/change-tracking.md @@ -0,0 +1,9 @@ +--- +title: "Change tracking in branch deployments" +displayed_sidebar: "dagsterPlus" +sidebar_position: 200 +sidebar_label: "Change tracking" +unlisted: true +--- + +# Change tracking in branch deployments diff --git a/docs/docs-beta/docs/dagster-plus/features/branch-deployments/dagster-cloud-cli.md b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/dagster-cloud-cli.md new file mode 100644 index 0000000000000..76615d4e4ad5d --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/dagster-cloud-cli.md @@ -0,0 +1,9 @@ +--- +title: "Branch deployments & the dagster-cloud CLI" +displayed_sidebar: "dagsterPlus" +sidebar_position: 300 +sidebar_label: "dagster-cloud CLI" +unlisted: true +--- + +# Use branch deployments with the dagster-cloud CLI diff --git a/docs/docs-beta/docs/dagster-plus/features/branch-deployments/index.mdx b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/index.mdx new file mode 100644 index 0000000000000..bdb1ae1d19be9 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/index.mdx @@ -0,0 +1,11 @@ +--- +title: "Branch deployments (CI)" +displayed_sidebar: "dagsterPlus" +sidebar_position: 20 +--- + +# Branch deployments (CI) + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/setting-up-branch-deployments.md b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/setting-up-branch-deployments.md similarity index 98% rename from docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/setting-up-branch-deployments.md rename to docs/docs-beta/docs/dagster-plus/features/branch-deployments/setting-up-branch-deployments.md index 9e47800077f89..39ef33b1362cb 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/branch-deployments/setting-up-branch-deployments.md +++ b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/setting-up-branch-deployments.md @@ -1,9 +1,8 @@ --- -title: "Setting up Branch Deployments" +title: "Setting up branch deployments" displayed_sidebar: "dagsterPlus" -sidebar_position: 1 +sidebar_position: 100 sidebar_label: "Setting up branch deployments" -toc_max_heading_level: 2 --- In this guide, we'll walk you through setting up Branch Deployments for a code location. Once you're finished, any time a PR is created or updated in your repository, it will automatically create or update an associated branch deployment in Dagster+. @@ -445,5 +444,5 @@ dagster-cloud deployment add-location \ ## Next steps -- Learn more about [Branch Deployments](/dagster-plus/deployment/branch-deployments) -- Learn how to [Track changes on a Branch Deployment](/dagster-plus/deployment/branch-deployments/change-tracking) +- Learn more about [Branch Deployments](/dagster-plus/features/branch-deployments/index.mdx) +- Learn how to [Track changes on a Branch Deployment](/dagster-plus/features/branch-deployments/change-tracking) diff --git a/docs/docs-beta/docs/dagster-plus/features/branch-deployments/testing.md b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/testing.md new file mode 100644 index 0000000000000..aaf141806d021 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/features/branch-deployments/testing.md @@ -0,0 +1,7 @@ +--- +title: "Testing against production with branch deployments" +unlisted: true +sidebar_position: 400 +--- + +# Testing against production with branch deployments \ No newline at end of file diff --git a/docs/docs-beta/docs/dagster-plus/data-catalog.md b/docs/docs-beta/docs/dagster-plus/features/data-catalog.md similarity index 76% rename from docs/docs-beta/docs/dagster-plus/data-catalog.md rename to docs/docs-beta/docs/dagster-plus/features/data-catalog.md index ac71fc43f8cde..41afe2b6a7bed 100644 --- a/docs/docs-beta/docs/dagster-plus/data-catalog.md +++ b/docs/docs-beta/docs/dagster-plus/features/data-catalog.md @@ -1,6 +1,7 @@ --- title: "Data catalog" unlisted: true +sidebar_position: 200 --- # Dagster+ data catalog diff --git a/docs/docs-beta/docs/dagster-plus/insights/asset-metadata.md b/docs/docs-beta/docs/dagster-plus/features/insights/asset-metadata.md similarity index 94% rename from docs/docs-beta/docs/dagster-plus/insights/asset-metadata.md rename to docs/docs-beta/docs/dagster-plus/features/insights/asset-metadata.md index 50381d318fd68..ba05db163f29d 100644 --- a/docs/docs-beta/docs/dagster-plus/insights/asset-metadata.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/asset-metadata.md @@ -1,7 +1,7 @@ --- title: "Integrate asset metadata into Dagster+ Insights" sidebar_label: "Integrate asset metadata" -sidebar_position: 1 +sidebar_position: 100 --- Out of the box, Dagster+ Insights gives you visibility into a variety of common metrics across your data platform. @@ -21,7 +21,7 @@ You'll need one or more assets that emit the same metadata key at run time. Insi are most valuable when you have multiple assets that emit the same kind of metadata, such as such as the number of rows processed or the size of a file uploaded to object storage. -Follow [the metadata guide](/guides/metadata#runtime-metadata) to add numeric metadata +Follow [the metadata guide](/guides/build/create-a-pipeline/metadata#runtime-metadata) to add numeric metadata to your asset materializations. ## Step 2: Enable viewing your metadata in Dagster+ Insights diff --git a/docs/docs-beta/docs/dagster-plus/insights/export-metrics.md b/docs/docs-beta/docs/dagster-plus/features/insights/export-metrics.md similarity index 86% rename from docs/docs-beta/docs/dagster-plus/insights/export-metrics.md rename to docs/docs-beta/docs/dagster-plus/features/insights/export-metrics.md index 79a5b16c615b4..3651b20fdbf4e 100644 --- a/docs/docs-beta/docs/dagster-plus/insights/export-metrics.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/export-metrics.md @@ -1,7 +1,7 @@ --- title: "Export metrics from Dagster+ Insights" sidebar_label: "Export metrics" -sidebar_position: 6 +sidebar_position: 200 unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/insights/google-bigquery.md b/docs/docs-beta/docs/dagster-plus/features/insights/google-bigquery.md similarity index 96% rename from docs/docs-beta/docs/dagster-plus/insights/google-bigquery.md rename to docs/docs-beta/docs/dagster-plus/features/insights/google-bigquery.md index 072e39a7a9ea1..1413a49ba0595 100644 --- a/docs/docs-beta/docs/dagster-plus/insights/google-bigquery.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/google-bigquery.md @@ -1,7 +1,7 @@ --- title: "Track Google BigQuery usage with Dagster+ Insights" sidebar_label: "Google BigQuery" -sidebar_position: 2 +sidebar_position: 400 --- Dagster allows you to track external metrics, such as BigQuery usage in the Insights UI. Out of the box integrations are provided to capture query runtime and billed usage, and associate them with the relevant assets or jobs. @@ -15,7 +15,7 @@ The BigQuery cost metric is based off of the bytes billed for queries executed w To use these features, you will need: - A Dagster+ account on the **Pro** plan -- Access to the [Dagster+ Insights feature](/dagster-plus/insights) +- Access to the [Dagster+ Insights feature](/dagster-plus/features/insights) - BigQuery credentials which have access to the **`INFORMATION_SCHEMA.JOBS`** table, such as a BigQuery Resource viewer role. - For more information, see the [BigQuery Documentation](https://cloud.google.com/bigquery/docs/information-schema-jobs) - The following packages installed: diff --git a/docs/docs-beta/docs/dagster-plus/insights.md b/docs/docs-beta/docs/dagster-plus/features/insights/index.md similarity index 89% rename from docs/docs-beta/docs/dagster-plus/insights.md rename to docs/docs-beta/docs/dagster-plus/features/insights/index.md index 6819907cada01..a36a2e9513a76 100644 --- a/docs/docs-beta/docs/dagster-plus/insights.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/index.md @@ -1,6 +1,7 @@ --- title: 'Insights' description: 'Visiblity into historical usage, cost, and metadata.' +sidebar_position: 10 --- # Dagster+ Insights @@ -14,7 +15,7 @@ Visualizations are built into the Dagster+ UI, allowing you to explore metrics f - [Explore usage trends in your Dagster pipelines](#explore-dagsters-built-in-metrics) - [Integrate additional metrics](#integrate-metrics), like data warehouse cost or your own custom metadata - [Export metrics](#export-metrics) from Dagster+ -- [Create alerts](/dagster-plus/deployment/alerts) based off of Insights metrics TODO: write this alerts section +- [Create alerts](/dagster-plus/features/alerts) based off of Insights metrics TODO: write this alerts section
Prerequisites @@ -62,10 +63,10 @@ How long historical Insights data is retained depends on your Dagster+ plan: Users on the Pro plan can integration other metrics into Insights, such as asset materialization metadata or Snowflake credits. Insights supports the following additional metrics: -- **Asset materialization metadata.** Refer to the [Using asset metadata with Dagster+ Insights guide](/dagster-plus/insights/asset-metadata) for more info. -- [**Google BigQuery usage**](/dagster-plus/insights/google-bigquery) generated by either queries made to BigQuery resources or using dbt to materialize tables -- [**Snowflake usage**](/dagster-plus/insights/snowflake) generated by either queries made to Snowflake resources or using dbt to materialize tables +- **Asset materialization metadata.** Refer to the [Using asset metadata with Dagster+ Insights guide](/dagster-plus/features/insights/asset-metadata) for more info. +- [**Google BigQuery usage**](/dagster-plus/features/insights/google-bigquery) generated by either queries made to BigQuery resources or using dbt to materialize tables +- [**Snowflake usage**](/dagster-plus/features/insights/snowflake) generated by either queries made to Snowflake resources or using dbt to materialize tables ### Export metrics -Metrics in Dagster+ Insights can be exported using a GraphQL API endpoint. Refer to the [Exporting Insights metrics from Dagster+ guide](/dagster-plus/insights/export-metrics) for details. +Metrics in Dagster+ Insights can be exported using a GraphQL API endpoint. Refer to the [Exporting Insights metrics from Dagster+ guide](/dagster-plus/features/insights/export-metrics) for details. diff --git a/docs/docs-beta/docs/dagster-plus/insights/snowflake.md b/docs/docs-beta/docs/dagster-plus/features/insights/snowflake.md similarity index 97% rename from docs/docs-beta/docs/dagster-plus/insights/snowflake.md rename to docs/docs-beta/docs/dagster-plus/features/insights/snowflake.md index 5d5674652974e..ae4af2b9f1777 100644 --- a/docs/docs-beta/docs/dagster-plus/insights/snowflake.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/snowflake.md @@ -1,7 +1,7 @@ --- title: "Track Snowflake usage with Dagster+ Insights" sidebar_label: "Snowflake" -sidebar_position: 2 +sidebar_position: 300 --- Dagster allows you to track external metrics, such as Snowflake usage, in the Insights UI. Out of the box integrations are provided to capture query runtime and billed usage, and associate them with the relevant assets or jobs. @@ -11,7 +11,7 @@ Dagster allows you to track external metrics, such as Snowflake usage, in the In To use these features, you will need: - A Dagster+ account on the **Pro** plan -- Access to the [Dagster+ Insights feature](/dagster-plus/insights) +- Access to the [Dagster+ Insights feature](/dagster-plus/features/insights) - Snowflake credentials which have access to the **`snowflake.account_usage.query_history`**. - For more information, see the [Snowflake Documentation](https://docs.snowflake.com/en/sql-reference/account-usage#enabling-the-snowflake-database-usage-for-other-roles) - The following packages installed: diff --git a/docs/docs-beta/docs/dagster-plus/saved-views.md b/docs/docs-beta/docs/dagster-plus/features/saved-views.md similarity index 95% rename from docs/docs-beta/docs/dagster-plus/saved-views.md rename to docs/docs-beta/docs/dagster-plus/features/saved-views.md index 3f3c0a7bc1ef4..fc06f04032485 100644 --- a/docs/docs-beta/docs/dagster-plus/saved-views.md +++ b/docs/docs-beta/docs/dagster-plus/features/saved-views.md @@ -1,6 +1,7 @@ --- title: "Saving asset filters as catalog views" displayed_sidebar: "dagsterPlus" +sidebar_position: 100 --- # Saving asset filters as catalog views @@ -19,7 +20,7 @@ In this guide, you'll learn how to create, access, and share catalog views with Prerequisites - **Organization Admin**, **Admin**, or **Editor** permissions on Dagster+ -- Familiarity with [Assets](/concepts/assets) and [Asset metadata](/guides/metadata) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx and [Asset metadata](/guides/build/create-a-pipeline/metadata)
diff --git a/docs/docs-beta/docs/dagster-plus/getting-started.md b/docs/docs-beta/docs/dagster-plus/getting-started.md index afe3e4f4666bc..52b79f7b13418 100644 --- a/docs/docs-beta/docs/dagster-plus/getting-started.md +++ b/docs/docs-beta/docs/dagster-plus/getting-started.md @@ -3,7 +3,7 @@ title: "Getting started with Dagster+" displayed_sidebar: "dagsterPlus" --- -# Get started with Dagster+ +# Getting started with Dagster+ First [create a Dagster+ organization](https://dagster.plus/signup). Note: you can sign up with: - a Google email address @@ -58,4 +58,4 @@ Refer to the guide for [adding a code location](/dagster-plus/deployment/code-lo ## Next steps -Your Dagster+ account is automatically enrolled in a trial. You can [pick your plan type and enter your billing information](/dagster-plus/settings), or [contact the Dagster team](https://dagster.io/contact) if you need support or want to evaluate the Dagster+ Pro plan. +Your Dagster+ account is automatically enrolled in a trial. You can [pick your plan type and enter your billing information](/dagster-plus/deployment/dagster-plus-settings), or [contact the Dagster team](https://dagster.io/contact) if you need support or want to evaluate the Dagster+ Pro plan. diff --git a/docs/docs-beta/docs/dagster-plus/whats-dagster-plus.md b/docs/docs-beta/docs/dagster-plus/index.md similarity index 60% rename from docs/docs-beta/docs/dagster-plus/whats-dagster-plus.md rename to docs/docs-beta/docs/dagster-plus/index.md index 0fe6016026500..295a68941b406 100644 --- a/docs/docs-beta/docs/dagster-plus/whats-dagster-plus.md +++ b/docs/docs-beta/docs/dagster-plus/index.md @@ -1,10 +1,10 @@ --- -title: "What's Dagster+?" +title: "About Dagster+" displayed_sidebar: 'dagsterPlus' sidebar_position: 1 --- -# What's Dagster+? +# About Dagster+ Dagster+ is a managed orchestration platform built on top of Dagster's open source engine. @@ -20,11 +20,11 @@ Dagster+ Serverless is fully managed and your Dagster code executes in our envir In addition to managed infrastructure, Dagster+ also adds core capabilities on top of Dagster open source to enable teams building data platforms: -- [Insights](/dagster-plus/insights), a powerful tool for identifying trends in your data platform overtime, optimizing cost, and answering questions like "Why does it feel like our pipelines are taking longer this month?". -- [Alerts](/dagster-plus/deployment/alerts) to a variety of services like Slack, PagerDuty, and email to notify your team of failed runs, data quality issues, and violated SLAs. -- Authentication, [Role Based Access Control](/dagster-plus/access/rbac), and [Audit Logs](/dagster-plus/access/rbac/audit-logs) which help teams implement data mesh strategies while remaining compliant. -- [Data Catalog](/dagster-plus/data-catalog), a powerful search-first experience that builds off of Dagster's best-in-class lineage graph to include searching for assets, metadata, column lineage, and more. -- [Branch Deployments](/dagster-plus/deployment/branch-deployments) +- [Insights](/dagster-plus/features/insights), a powerful tool for identifying trends in your data platform overtime, optimizing cost, and answering questions like "Why does it feel like our pipelines are taking longer this month?". +- [Alerts](/dagster-plus/features/alerts) to a variety of services like Slack, PagerDuty, and email to notify your team of failed runs, data quality issues, and violated SLAs. +- Authentication, [Role Based Access Control](/dagster-plus/features/authentication-and-access-control/rbac), and [Audit Logs](/dagster-plus/features/authentication-and-access-control/rbac/audit-logs) which help teams implement data mesh strategies while remaining compliant. +- [Data Catalog](/dagster-plus/features/data-catalog), a powerful search-first experience that builds off of Dagster's best-in-class lineage graph to include searching for assets, metadata, column lineage, and more. +- [Branch Deployments](/dagster-plus/features/branch-deployments/index.mdx) Ready to [get started](/dagster-plus/getting-started)? @@ -33,5 +33,5 @@ Ready to [get started](/dagster-plus/getting-started)? - Learn more about Dagster+ [pricing and plan types](https://dagster.io/pricing) or [contact the Dagster team](https://dagster.io/contact) - Dagster+ includes support, [click here](https://dagster.io/support) to learn more. - Dagster+ is HIPAA compliant, SOC 2 Type II certified, and meets GDPR requirements. Learn more about Dagster+[ security](https://dagster.io/security). -- Migrate [from a Dagster open source deployment to Dagster+](/guides/self-hosted-to-dagster-plus) +- Migrate [from a Dagster open source deployment to Dagster+](/dagster-plus/deployment/migration/self-hosted-to-dagster-plus) - Dagster+ [status page](https://dagstercloud.statuspage.io/) diff --git a/docs/docs-beta/docs/getting-started/glossary.md b/docs/docs-beta/docs/getting-started/glossary.md new file mode 100644 index 0000000000000..8d0b5a8038b65 --- /dev/null +++ b/docs/docs-beta/docs/getting-started/glossary.md @@ -0,0 +1,19 @@ +--- +title: Glossary +sidebar_position: 30 +sidebar_label: Glossary +unlisted: true +--- + +# Glossary + +TODO - link to conceptual content about the following (and other key concepts if needed): + +* Assets +* Definitions +* Partitions (and backfills) +* Resources +* Schedules +* Sensors +* I/O managers? +* Ops and jobs (and graphs?) diff --git a/docs/docs-beta/docs/getting-started/installation.md b/docs/docs-beta/docs/getting-started/installation.md index 3a693ef2c99cd..e35319282fb79 100644 --- a/docs/docs-beta/docs/getting-started/installation.md +++ b/docs/docs-beta/docs/getting-started/installation.md @@ -72,4 +72,4 @@ If you encounter any issues during the installation process: ## Next steps - Get up and running with your first Dagster project in the [Quickstart](/getting-started/quickstart) -- Learn to [create data assets in Dagster](/guides/data-assets) +- Learn to [create data assets in Dagster](/guides/build/create-a-pipeline/data-assets) diff --git a/docs/docs-beta/docs/getting-started/quickstart.md b/docs/docs-beta/docs/getting-started/quickstart.md index 9e9eef0529090..afb15e582ed45 100644 --- a/docs/docs-beta/docs/getting-started/quickstart.md +++ b/docs/docs-beta/docs/getting-started/quickstart.md @@ -154,4 +154,4 @@ id,name,age,city,age_group Congratulations! You've just built and run your first pipeline with Dagster. Next, you can: - Continue with the [ETL pipeline tutorial](/tutorial/tutorial-etl) to learn how to build a more complex ETL pipeline -- Learn how to [Think in assets](/concepts/assets/thinking-in-assets) +- Learn how to [Think in assets](/guides/build/assets-concepts/index.md) diff --git a/docs/docs-beta/docs/concepts/automation.md b/docs/docs-beta/docs/guides/automate/about-automation.md similarity index 86% rename from docs/docs-beta/docs/concepts/automation.md rename to docs/docs-beta/docs/guides/automate/about-automation.md index 58a8cfafe497f..70f0f94643e50 100644 --- a/docs/docs-beta/docs/concepts/automation.md +++ b/docs/docs-beta/docs/guides/automate/about-automation.md @@ -1,14 +1,15 @@ --- title: About Automation +unlisted: true --- There are several ways to automate the execution of your data pipelines with Dagster. -The first system, and the most basic, is the [Schedule](/guides/schedules), which responds to time. +The first system, and the most basic, is the [Schedule](/guides/automate/schedules), which responds to time. -[Sensors](/guides/sensors) are like schedules, but they respond to an external event defined by the user. +[Sensors](/guides/automate/sensors) are like schedules, but they respond to an external event defined by the user. -[Asset Sensors](/guides/asset-sensors) are a special case of sensor that responds to changes in asset materialization +[Asset Sensors](/guides/automate/asset-sensors) are a special case of sensor that responds to changes in asset materialization as reported by the Event Log. Finally, the Declarative Automation system is a @@ -23,7 +24,7 @@ as the schedule is processed. Schedules were one of the first types of automation in Dagster, created before the introduction of Software-Defined Assets. As such, you may find that many of the examples can seem foreign if you are used to only working within the asset framework. -For more on how assets and ops inter-relate, read about [Assets and Ops](/concepts/assets#assets-and-ops) +For more on how assets and ops inter-relate, read about [Assets and Ops](/guides/build/assets-concepts#assets-and-ops) The `dagster-daemon` process is responsible for submitting runs by checking each schedule at a regular interval to determine if it's time to execute the underlying job. diff --git a/docs/docs-beta/docs/guides/asset-sensors.md b/docs/docs-beta/docs/guides/automate/asset-sensors.md similarity index 93% rename from docs/docs-beta/docs/guides/asset-sensors.md rename to docs/docs-beta/docs/guides/automate/asset-sensors.md index 1685d0714218e..2c71bdaca5bd3 100644 --- a/docs/docs-beta/docs/guides/asset-sensors.md +++ b/docs/docs-beta/docs/guides/automate/asset-sensors.md @@ -1,6 +1,6 @@ --- title: Triggering cross-job dependencies with Asset Sensors -sidebar_position: 30 +sidebar_position: 300 sidebar_label: Cross-job dependencies --- @@ -13,8 +13,8 @@ This guide covers the most common use cases for asset sensors, such as defining To follow this guide, you'll need: -- Familiarity with [Assets](/concepts/assets) -- Familiarity with [Ops and Jobs](/concepts/ops-jobs) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx +- Familiarity with [Ops and Jobs](/guides/build/ops-jobs) @@ -101,5 +101,5 @@ The following example uses a `@multi_asset_sensor` to monitor multiple assets an ## Next steps -- Learn more about asset sensors in [Understanding Automation](/concepts/automation) -- Explore [Declarative Automation](/concepts/automation/declarative-automation) as an alternative to asset sensors +- Learn more about asset sensors in [Understanding Automation](/guides/automate) +- Explore [Declarative Automation](/guides/automate/declarative-automation) as an alternative to asset sensors diff --git a/docs/docs-beta/docs/guides/declarative-automation.md b/docs/docs-beta/docs/guides/automate/declarative-automation.md similarity index 81% rename from docs/docs-beta/docs/guides/declarative-automation.md rename to docs/docs-beta/docs/guides/automate/declarative-automation.md index 806b0a380df5f..9a26ff8d2a427 100644 --- a/docs/docs-beta/docs/guides/declarative-automation.md +++ b/docs/docs-beta/docs/guides/automate/declarative-automation.md @@ -2,4 +2,5 @@ title: "Declarative automation" sidebar_label: "Declarative automation" unlisted: true +sidebar_position: 400 --- diff --git a/docs/docs-beta/docs/guides/automation.md b/docs/docs-beta/docs/guides/automate/index.md similarity index 91% rename from docs/docs-beta/docs/guides/automation.md rename to docs/docs-beta/docs/guides/automate/index.md index 55b476e65bb6b..a10399054e850 100644 --- a/docs/docs-beta/docs/guides/automation.md +++ b/docs/docs-beta/docs/guides/automate/index.md @@ -1,9 +1,8 @@ --- -title: "Automating Pipelines" +title: "Automating pipelines" description: Learn how to automate your data pipelines. -last_update: - date: 2024-08-12 - author: Pedram Navid +sidebar_position: 40 +sidebar_class_name: hidden --- Automation is key to building reliable, efficient data pipelines. This guide provides a simplified overview of the main ways to automate processes in Dagster, helping you choose the right method for your needs. You will find links to more detailed guides for each method below. @@ -13,8 +12,8 @@ Automation is key to building reliable, efficient data pipelines. This guide pro Before continuing, you should be familiar with: -- [Asset definitions](/concepts/assets) -- [Jobs](/concepts/ops-jobs) +- [Asset definitions](/guides/build/assets-concepts/index.mdx +- [Jobs](/guides/build/ops-jobs) @@ -49,7 +48,7 @@ Use this table to help guide your decision: ## Schedules Schedules allow you to run jobs at specified times, like "every Monday at 9 AM" or "daily at midnight." -A schedule combines a selection of assets, known as a [Job](/concepts/ops-jobs), and a [cron expression](https://en.wikipedia.org/wiki/Cron) +A schedule combines a selection of assets, known as a [Job](/guides/build/ops-jobs), and a [cron expression](https://en.wikipedia.org/wiki/Cron) to define when the job should be run. To make creating cron expressions easier, you can use an online tool like [Crontab Guru](https://crontab.guru/). @@ -59,9 +58,9 @@ To make creating cron expressions easier, you can use an online tool like [Cront - You need to run jobs at regular intervals - You want basic time-based automation -For examples of how to create schedules, see [How-To Use Schedules](/guides/schedules). +For examples of how to create schedules, see [How-To Use Schedules](/guides/automate/schedules). -For more information about how Schedules work, see [About Schedules](/concepts/schedules). +For more information about how Schedules work, see [About Schedules](/guides/automate/schedules). ## Sensors @@ -69,16 +68,16 @@ Sensors allow you to trigger runs based on events or conditions that you define, You must provide a function that the sensor will use to determine if it should trigger a run. -Like schedules, sensors operate on a selection of assets, known as [Jobs](/concepts/ops-jobs) and can either start a pipeline through a Run or log a reason for not starting a pipeline using a SkipReason. +Like schedules, sensors operate on a selection of assets, known as [Jobs](/guides/build/ops-jobs) and can either start a pipeline through a Run or log a reason for not starting a pipeline using a SkipReason. ### When to use sensors - You need event-driven automation - You want to react to changes in external systems -For more examples of how to create sensors, see the [How-To Use Sensors](/guides/sensors) guide. +For more examples of how to create sensors, see the [How-To Use Sensors](/guides/automate/sensors) guide. -For more information about how sensors work, see the [About Sensors](/concepts/sensors) concept page. +For more information about how sensors work, see the [About Sensors](/guides/automate/sensors) concept page. ## Asset sensors @@ -89,7 +88,7 @@ Asset Sensors trigger jobs when specified assets are materialized, allowing you - You need to trigger jobs based on asset materializations - You want to create dependencies between different jobs or code locations -For more examples of how to create asset sensors, see the [How-To Use Asset Sensors](/guides/asset-sensors) guide. +For more examples of how to create asset sensors, see the [How-To Use Asset Sensors](/guides/automate/asset-sensors) guide. ## Declarative Automation diff --git a/docs/docs-beta/docs/guides/schedules.md b/docs/docs-beta/docs/guides/automate/schedules.md similarity index 78% rename from docs/docs-beta/docs/guides/schedules.md rename to docs/docs-beta/docs/guides/automate/schedules.md index 9355498a7d1df..3d749758ecb87 100644 --- a/docs/docs-beta/docs/guides/schedules.md +++ b/docs/docs-beta/docs/guides/automate/schedules.md @@ -1,7 +1,7 @@ --- title: "Schedule cron-based pipelines" sidebar_label: "Schedules" -sidebar_position: 10 +sidebar_position: 100 --- Schedules enable automated execution of jobs at specified intervals. These intervals can range from common frequencies like hourly, daily, or weekly, to more intricate patterns defined using cron expressions. @@ -11,8 +11,8 @@ Schedules enable automated execution of jobs at specified intervals. These inter To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/concepts/assets) -- Familiarity with [Ops and Jobs](/concepts/ops-jobs) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx +- Familiarity with [Ops and Jobs](/guides/build/ops-jobs/index.md) ## Basic schedule @@ -41,7 +41,7 @@ If using partitions and jobs, you can create a schedule using the partition with -If you have a [partitioned asset](/guides/partitioning) and job: +If you have a [partitioned asset](/guides/build/create-a-pipeline/partitioning) and job: @@ -59,6 +59,6 @@ If you have a partitioned op job: By understanding and effectively using these automation methods, you can build more efficient data pipelines that respond to your specific needs and constraints: -- Learn more about schedules in [Understanding automation](/concepts/automation) -- React to events with [sensors](/guides/sensors) -- Explore [Declarative Automation](/concepts/automation/declarative-automation) as an alternative to schedules +- Learn more about schedules in [Understanding automation](/guides/automate/index.md) +- React to events with [sensors](/guides/automate/sensors) +- Explore [Declarative Automation](/guides/automate/declarative-automation) as an alternative to schedules diff --git a/docs/docs-beta/docs/guides/sensors.md b/docs/docs-beta/docs/guides/automate/sensors.md similarity index 89% rename from docs/docs-beta/docs/guides/sensors.md rename to docs/docs-beta/docs/guides/automate/sensors.md index 60150744ab35f..71be08e940220 100644 --- a/docs/docs-beta/docs/guides/sensors.md +++ b/docs/docs-beta/docs/guides/automate/sensors.md @@ -1,13 +1,13 @@ --- title: Creating event-based pipelines with sensors sidebar_label: Event triggers -sidebar_position: 20 +sidebar_position: 200 --- Sensors enable you to trigger Dagster runs in response to events from external systems. They run at regular intervals, either triggering a run or explaining why a run was skipped. For example, you can trigger a run when a new file is added to an Amazon S3 bucket or when a database row is updated. :::tip -An alternative to polling with sensors is to push events to Dagster using the [Dagster API](/guides/automation#graphql-endpoint). +An alternative to polling with sensors is to push events to Dagster using the [Dagster API](/guides/automate#graphql-endpoint). :::
@@ -15,8 +15,8 @@ An alternative to polling with sensors is to push events to Dagster using the [D To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/concepts/assets) -- Familiarity with [Ops and Jobs](/concepts/ops-jobs) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx +- Familiarity with [Ops and Jobs](/guides/build/ops-jobs)
## Basic sensor @@ -76,6 +76,6 @@ If you want to be able to reset a sensor's cursor, don't set `run_key`s on `RunR By understanding and effectively using these automation methods, you can build more efficient data pipelines that respond to your specific needs and constraints. -- Run pipelines on a [schedule](/guides/schedules) -- Trigger cross-job dependencies with [asset sensors](/guides/asset-sensors) -- Explore [Declarative Automation](/concepts/automation/declarative-automation) as an alternative to sensors +- Run pipelines on a [schedule](/guides/automate/schedules) +- Trigger cross-job dependencies with [asset sensors](/guides/automate/asset-sensors) +- Explore [Declarative Automation](/guides/automate/declarative-automation) as an alternative to sensors diff --git a/docs/docs-beta/docs/guides/asset-dependencies.md b/docs/docs-beta/docs/guides/build/assets-concepts/asset-dependencies.md similarity index 96% rename from docs/docs-beta/docs/guides/asset-dependencies.md rename to docs/docs-beta/docs/guides/build/assets-concepts/asset-dependencies.md index 421a61a08fece..20495ba819d7f 100644 --- a/docs/docs-beta/docs/guides/asset-dependencies.md +++ b/docs/docs-beta/docs/guides/build/assets-concepts/asset-dependencies.md @@ -1,10 +1,7 @@ --- title: Pass data between assets description: Learn how to pass data between assets in Dagster -sidebar_position: 30 -last_update: - date: 2024-08-11 - author: Pedram Navid +sidebar_position: 200 --- In Dagster, assets are the building blocks of your data pipeline and it's common to want to pass data between them. This guide will help you understand how to pass data between assets. @@ -57,7 +54,7 @@ I/O managers handle: 1. **Input**: Reading data from storage and loading it into memory for use by dependent assets. 2. **Output**: Writing data to the configured storage location. -For a deeper understanding of I/O managers, check out the [Understanding I/O managers](/concepts/io-managers) guide. +For a deeper understanding of I/O managers, check out the [Understanding I/O managers](/guides/build/configure/io-managers) guide. @@ -94,7 +91,7 @@ Consider this example: This example downloads a zip file from Google Drive, unzips it, and loads the data into a Pandas DataFrame. It relies on each asset running on the same file system to perform these operations. -The assets are modeled as tasks, rather than as data assets. For more information on the difference between tasks and data assets, check out the [Thinking in Assets](/concepts/assets/thinking-in-assets) guide. +The assets are modeled as tasks, rather than as data assets. For more information on the difference between tasks and data assets, check out the [assets guide](/guides/build/assets-concepts/index.md). In this refactor, the `download_files`, `unzip_files`, and `load_data` assets are combined into a single asset, `my_dataset`. This asset downloads the files, unzips them, and loads the data into a data warehouse. diff --git a/docs/docs-beta/docs/guides/asset-factories-with-deps.md b/docs/docs-beta/docs/guides/build/assets-concepts/asset-factories-with-deps.md similarity index 96% rename from docs/docs-beta/docs/guides/asset-factories-with-deps.md rename to docs/docs-beta/docs/guides/build/assets-concepts/asset-factories-with-deps.md index 3aec89d72554e..07de1116baee6 100644 --- a/docs/docs-beta/docs/guides/asset-factories-with-deps.md +++ b/docs/docs-beta/docs/guides/build/assets-concepts/asset-factories-with-deps.md @@ -1,6 +1,6 @@ --- title: 'Programmatically defining dependencies using asset factories' -sidebar_position: 60 +sidebar_position: 400 sidebar_label: 'Asset Factories (2)' --- @@ -13,7 +13,7 @@ Using an asset factory reduces complexity and creates a pluggable entry point to
Prerequisites -This guide builds upon the concepts in the [asset factories](/guides/asset-factories) tutorial. +This guide builds upon the concepts in the [asset factories](/guides/build/configure/asset-factories) tutorial.
--- diff --git a/docs/docs-beta/docs/concepts/assets/asset-materialization.md b/docs/docs-beta/docs/guides/build/assets-concepts/asset-materialization.md similarity index 78% rename from docs/docs-beta/docs/concepts/assets/asset-materialization.md rename to docs/docs-beta/docs/guides/build/assets-concepts/asset-materialization.md index 01ffb57146241..4bb472f5c7884 100644 --- a/docs/docs-beta/docs/concepts/assets/asset-materialization.md +++ b/docs/docs-beta/docs/guides/build/assets-concepts/asset-materialization.md @@ -1,6 +1,6 @@ --- title: "Asset materialization" -sidebar_position: 20 +sidebar_position: 300 unlisted: true --- diff --git a/docs/docs-beta/docs/concepts/assets/asset-metadata.md b/docs/docs-beta/docs/guides/build/assets-concepts/asset-metadata.md similarity index 74% rename from docs/docs-beta/docs/concepts/assets/asset-metadata.md rename to docs/docs-beta/docs/guides/build/assets-concepts/asset-metadata.md index 5f707d5641247..276b764359664 100644 --- a/docs/docs-beta/docs/concepts/assets/asset-metadata.md +++ b/docs/docs-beta/docs/guides/build/assets-concepts/asset-metadata.md @@ -1,6 +1,6 @@ --- title: "Asset metadata" -sidebar_position: 40 +sidebar_position: 100 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/build/assets-concepts/index.md b/docs/docs-beta/docs/guides/build/assets-concepts/index.md new file mode 100644 index 0000000000000..9e2f302b468a0 --- /dev/null +++ b/docs/docs-beta/docs/guides/build/assets-concepts/index.md @@ -0,0 +1,11 @@ +--- +title: "Assets concepts" +sidebar_position: 40 +sidebar_class_name: hidden +--- + +TODO + +## Assets and ops + +Assets and ops are two different concepts in Dagster. For more information, see "[Ops vs assets](/guides/build/ops-jobs/ops-vs-assets)". diff --git a/docs/docs-beta/docs/guides/selection-syntax.md b/docs/docs-beta/docs/guides/build/assets-concepts/selection-syntax.md similarity index 99% rename from docs/docs-beta/docs/guides/selection-syntax.md rename to docs/docs-beta/docs/guides/build/assets-concepts/selection-syntax.md index f87cf5bd231f4..1676ed5af6a80 100644 --- a/docs/docs-beta/docs/guides/selection-syntax.md +++ b/docs/docs-beta/docs/guides/build/assets-concepts/selection-syntax.md @@ -1,6 +1,6 @@ --- title: 'Asset selection syntax' -sidebar_position: 70 +sidebar_position: 500 sidebar_label: 'Asset selection syntax' --- diff --git a/docs/docs-beta/docs/guides/backfill.md b/docs/docs-beta/docs/guides/build/backfill.md similarity index 77% rename from docs/docs-beta/docs/guides/backfill.md rename to docs/docs-beta/docs/guides/build/backfill.md index 220d06051df23..8dbffbfc8197e 100644 --- a/docs/docs-beta/docs/guides/backfill.md +++ b/docs/docs-beta/docs/guides/build/backfill.md @@ -1,6 +1,6 @@ --- title: Backfilling data -sidebar_position: 60 sidebar_label: "Backfilling" unlisted: true +sidebar_position: 200 --- diff --git a/docs/docs-beta/docs/guides/apis.md b/docs/docs-beta/docs/guides/build/configure/apis.md similarity index 95% rename from docs/docs-beta/docs/guides/apis.md rename to docs/docs-beta/docs/guides/build/configure/apis.md index 349b0aba3ebc7..005a19ae99c50 100644 --- a/docs/docs-beta/docs/guides/apis.md +++ b/docs/docs-beta/docs/guides/build/configure/apis.md @@ -1,6 +1,6 @@ --- title: Connecting to APIs -sidebar_position: 20 +sidebar_position: 600 sidebar_label: API connections --- @@ -11,8 +11,8 @@ When building a data pipeline, you'll likely need to connect to several external To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/concepts/assets) -- Familiarity with [Resources](/concepts/resources) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx +- Familiarity with [Resources](/guides/build/configure/resources) - To install the `requests` library: ```bash pip install requests diff --git a/docs/docs-beta/docs/guides/asset-factories.md b/docs/docs-beta/docs/guides/build/configure/asset-factories.md similarity index 95% rename from docs/docs-beta/docs/guides/asset-factories.md rename to docs/docs-beta/docs/guides/build/configure/asset-factories.md index df6c615f91df4..9f7c3e687cb37 100644 --- a/docs/docs-beta/docs/guides/asset-factories.md +++ b/docs/docs-beta/docs/guides/build/configure/asset-factories.md @@ -1,6 +1,6 @@ --- title: 'Creating domain-specific languages with asset factories' -sidebar_position: 60 +sidebar_position: 300 sidebar_label: 'Asset factories' --- @@ -19,8 +19,8 @@ The asset factory pattern can solve both of these problems. To follow the steps in this guide, you'll need: - Familiarity with: - - [Assets](/guides/data-assets) - - [Resources](/concepts/resources) + - [Assets](/guides/build/create-a-pipeline/data-assets) + - [Resources](/guides/build/configure/resources) - SQL, YAML and Amazon Web Services (AWS) S3 - [Pydantic](https://docs.pydantic.dev/latest/) and [Jinja2](https://jinja.palletsprojects.com/en/3.1.x/) - A Python virtual environment with the following dependencies installed: diff --git a/docs/docs-beta/docs/guides/authentication.md b/docs/docs-beta/docs/guides/build/configure/authentication.md similarity index 72% rename from docs/docs-beta/docs/guides/authentication.md rename to docs/docs-beta/docs/guides/build/configure/authentication.md index d089c393abc7d..6dc4fb4c0f305 100644 --- a/docs/docs-beta/docs/guides/authentication.md +++ b/docs/docs-beta/docs/guides/build/configure/authentication.md @@ -1,5 +1,5 @@ --- title: Authenticating to a resource -sidebar_position: 60 +sidebar_position: 700 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/configuring-assets.md b/docs/docs-beta/docs/guides/build/configure/configuring-assets.md similarity index 80% rename from docs/docs-beta/docs/guides/configuring-assets.md rename to docs/docs-beta/docs/guides/build/configure/configuring-assets.md index 24ae1f74b4d1d..da028f5761cc6 100644 --- a/docs/docs-beta/docs/guides/configuring-assets.md +++ b/docs/docs-beta/docs/guides/build/configure/configuring-assets.md @@ -1,7 +1,7 @@ --- title: Configuring assets sidebar_label: Asset runs -sidebar_position: 50 +sidebar_position: 100 --- The Dagster UI is commonly used to manually materialize assets, backfill historical data, debug a production issue, or some other one-off task. @@ -13,7 +13,7 @@ You'll often want to be able to adjust parameters when materializing assets, whi To follow the steps in this guide, you'll need familiarity with: -- [Assets](/guides/data-assets) +- [Assets](/guides/build/create-a-pipeline/data-assets) - [Pydantic](https://docs.pydantic.dev/latest/) @@ -29,7 +29,7 @@ For example, you want to allow your team to change the lookback time window for ## Specifying config using the Dagster UI :::note -Run configurations reference an `op` which is the underlying compute associated with an asset. Refer to the [Ops vs Assets](/concepts/ops-jobs/ops-vs-assets) guide for more information. +Run configurations reference an `op` which is the underlying compute associated with an asset. Refer to the [Ops vs Assets](/guides/build/ops-jobs/ops-vs-assets) guide for more information. ::: When launching a run using the Launchpad in the UI, you can provide a run config file as YAML or JSON that overrides the default configuration for your asset. @@ -44,5 +44,5 @@ This will open the Launchpad, where you can scaffold the config, customize its v ## Next steps -- Learn more about Dagster [assets](/concepts/assets) -- Connect to external [APIs](/guides/apis) and [databases](/guides/databases) with resources +- Learn more about Dagster [assets](/guides/build/assets-concepts/index.mdx +- Connect to external [APIs](/guides/build/configure/apis) and [databases](/guides/build/configure/databases) with resources diff --git a/docs/docs-beta/docs/guides/databases.md b/docs/docs-beta/docs/guides/build/configure/databases.md similarity index 93% rename from docs/docs-beta/docs/guides/databases.md rename to docs/docs-beta/docs/guides/build/configure/databases.md index ab6f19ea279f3..84132316e9c87 100644 --- a/docs/docs-beta/docs/guides/databases.md +++ b/docs/docs-beta/docs/guides/build/configure/databases.md @@ -1,7 +1,7 @@ --- title: Connecting to databases description: How to configure resources to connect to databases -sidebar_position: 10 +sidebar_position: 500 sidebar_label: Database connections --- @@ -14,7 +14,7 @@ This guide demonstrates how to standardize database connections and customize th To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/guides/data-assets) +- Familiarity with [Assets](/guides/build/create-a-pipeline/data-assets) To run the examples in this guide, you'll need: @@ -59,5 +59,5 @@ You can also fetch environment variables using the `os` library. Dagster treats ## Next steps -- Explore how to use resources for [Connecting to APIs](/guides/apis) -- Go deeper into [Understanding Resources](/concepts/resources) \ No newline at end of file +- Explore how to use resources for [Connecting to APIs](/guides/build/configure/apis) +- Go deeper into [Understanding Resources](/guides/build/configure/resources) \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/build/configure/index.mdx b/docs/docs-beta/docs/guides/build/configure/index.mdx new file mode 100644 index 0000000000000..69dae1c64cc29 --- /dev/null +++ b/docs/docs-beta/docs/guides/build/configure/index.mdx @@ -0,0 +1,9 @@ +--- +title: "Configure" +description: "Configure assets, pipelines, and runs" +sidebar_position: 20 +--- + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/io-managers.md b/docs/docs-beta/docs/guides/build/configure/io-managers.md similarity index 94% rename from docs/docs-beta/docs/guides/io-managers.md rename to docs/docs-beta/docs/guides/build/configure/io-managers.md index 703ef990f5531..c64f4a909d995 100644 --- a/docs/docs-beta/docs/guides/io-managers.md +++ b/docs/docs-beta/docs/guides/build/configure/io-managers.md @@ -1,6 +1,6 @@ --- title: "Managing stored data with I/O managers" -sidebar_position: 50 +sidebar_position: 800 sidebar_label: "I/O managers" --- @@ -19,8 +19,8 @@ For assets that follow this pattern, an I/O manager can streamline the code that To follow the steps in this guide, you'll need familiarity with: -- [Assets](/concepts/assets) -- [Resources](/concepts/resources) +- [Assets](/guides/build/assets-concepts/index.mdx +- [Resources](/guides/build/configure/resources) ## Before you begin @@ -37,7 +37,7 @@ To follow the steps in this guide, you'll need familiarity with: - Your pipeline manages I/O on its own by using other libraries/tools that write to storage - Your assets won't fit in memory, such as a database table with billions of rows -As a general rule, if your pipeline becomes more complicated in order to use I/O managers, it's likely that I/O managers aren't a good fit. In these cases you should use `deps` to [define dependencies](/guides/asset-dependencies). +As a general rule, if your pipeline becomes more complicated in order to use I/O managers, it's likely that I/O managers aren't a good fit. In these cases you should use `deps` to [define dependencies](/guides/build/assets-concepts/asset-dependencies). ## Using I/O managers in assets \{#io-in-assets} @@ -84,5 +84,5 @@ Dagster offers built-in library implementations for I/O managers for popular dat ## Next steps -- Learn to [connect databases](/guides/databases) with resources -- Learn to [connect APIs](/guides/apis) with resources \ No newline at end of file +- Learn to [connect databases](/guides/build/configure/databases) with resources +- Learn to [connect APIs](/guides/build/configure/apis) with resources \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/managing-concurrency.md b/docs/docs-beta/docs/guides/build/configure/managing-concurrency.md similarity index 87% rename from docs/docs-beta/docs/guides/managing-concurrency.md rename to docs/docs-beta/docs/guides/build/configure/managing-concurrency.md index ef7e3af7b3015..59efe3ac2fe5d 100644 --- a/docs/docs-beta/docs/guides/managing-concurrency.md +++ b/docs/docs-beta/docs/guides/build/configure/managing-concurrency.md @@ -2,6 +2,7 @@ title: Managing concurrency of Dagster assets, jobs, and Dagster instances sidebar_label: Managing concurrency description: How to limit the number of runs a job, or assets for an instance of Dagster. +sidebar_position: 900 --- You often want to control the number of concurrent runs for a Dagster job, a specific asset, or for a type of asset or job. Limiting concurrency in your data pipelines can help prevent performance problems and downtime. @@ -10,8 +11,8 @@ You often want to control the number of concurrent runs for a Dagster job, a spe
Prerequisites -- Familiarity with [Assets](/concepts/assets) -- Familiarity with [Jobs and Ops](/concepts/ops-jobs) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx +- Familiarity with [Jobs and Ops](/guides/build/ops-jobs)
@@ -20,7 +21,7 @@ You often want to control the number of concurrent runs for a Dagster job, a spe * Dagster Core, add the following to your [dagster.yaml](/todo) -* In Dagster+, add the following to your [deployment settings](/dagster-plus/settings) +* In Dagster+, add the following to your [deployment settings](/dagster-plus/deployment/deployment-settings) ```yaml run_queue: @@ -49,7 +50,7 @@ You can control the number of assets or ops that are running concurrently within ## Limit how many of a certain type of op or asset can run across all runs -You can a limit for all ops or assets with a specific tag key or key-value pair. Ops or assets above that limit will be queued. Use `tag_concurrency_limits` in the job’s config, either in Python or using the Launchpad in the Dagster UI. +You can a limit for all ops or assets with a specific tag key or key-value pair. Ops or assets above that limit will be queued. Use `tag_concurrency_limits` in the job's config, either in Python or using the Launchpad in the Dagster UI. For example, you might want to limit the number of ops or assets that are running with a key of `database` across all runs (to limit the load on that database). @@ -148,17 +149,17 @@ The possible causes for runs remaining in `QUEUED` status depend on whether you' - If runs aren’t being dequeued in Dagster Open Source, the root cause is likely an issue with the Dagster daemon or the run queue configuration. + If runs aren't being dequeued in Dagster Open Source, the root cause is likely an issue with the Dagster daemon or the run queue configuration. #### Troubleshoot the Dagster daemon - * **Verify the Dagster daemon is set up and running.** In the Dagster UI, navigate to **Deployment > Daemons** and verify that the daemon is running. The **Run queue** should also be running. If you used [dagster dev](/guides/running-local-ui-development) to start the Dagster UI, the daemon should have been started for you. If the daemon isn’t running, proceed to step 2. + * **Verify the Dagster daemon is set up and running.** In the Dagster UI, navigate to **Deployment > Daemons** and verify that the daemon is running. The **Run queue** should also be running. If you used [dagster dev](/guides/deploy/deployment-options/running-local-ui-development) to start the Dagster UI, the daemon should have been started for you. If the daemon isn't running, proceed to step 2. * **Verify the Dagster daemon can access the same storage as the Dagster webserver process.** Both the webserver process and the Dagster daemon should access the same storage, meaning they should use the same `dagster.yaml`. Locally, this means both processes should have the same set `DAGSTER_HOME` environment variable. If you used dagster dev to start the Dagster UI, both processes should be using the same storage. Refer to the [Dagster Instance docs](/todo) for more information. #### Troubleshoot the run queue configuration If the daemon is running, runs may intentionally be left in the queue due to concurrency rules. To investigate: * **Check the output logged from the daemon process**, as this will include skipped runs. - * **Check the max_concurrent_runs setting in your instance’s dagster.yaml**. If set to 0, this may block the queue. You can check this setting in the Dagster UI by navigating to Deployment > Configuration and locating the run_queue.max_concurrent_runs setting. Refer to the Limiting overall runs section for more info. + * **Check the max_concurrent_runs setting in your instance's dagster.yaml**. If set to 0, this may block the queue. You can check this setting in the Dagster UI by navigating to Deployment > Configuration and locating the run_queue.max_concurrent_runs setting. Refer to the Limiting overall runs section for more info. * **Check the state of your run queue**. In some cases, the queue may be blocked by some number of in-progress runs. To view the status of your run queue, click **Runs** in the top navigation of the Dagster UI and then open the **Queued** and **In Progress** tabs. If there are queued or in-progress runs blocking the queue, you can terminate them to allow other runs to proceed. diff --git a/docs/docs-beta/docs/guides/resources.md b/docs/docs-beta/docs/guides/build/configure/resources.md similarity index 93% rename from docs/docs-beta/docs/guides/resources.md rename to docs/docs-beta/docs/guides/build/configure/resources.md index 76c3a5e6d1e06..632cd296f6091 100644 --- a/docs/docs-beta/docs/guides/resources.md +++ b/docs/docs-beta/docs/guides/build/configure/resources.md @@ -2,6 +2,7 @@ title: Using Resources to manage external systems sidebar_label: Resources unlisted: true +sidebar_position: 400 --- Dagster resources are objects that provide access to external systems, databases, or services. Resources are used to manage connections to external systems, and are used by Dagster ops and assets. diff --git a/docs/docs-beta/docs/guides/data-assets.md b/docs/docs-beta/docs/guides/build/create-a-pipeline/data-assets.md similarity index 93% rename from docs/docs-beta/docs/guides/data-assets.md rename to docs/docs-beta/docs/guides/build/create-a-pipeline/data-assets.md index 6d6540555d17a..5b37ffaaf1ec0 100644 --- a/docs/docs-beta/docs/guides/data-assets.md +++ b/docs/docs-beta/docs/guides/build/create-a-pipeline/data-assets.md @@ -1,7 +1,7 @@ --- title: 'Defining data assets with decorators' sidebar_label: 'Create Assets' -sidebar_position: 10 +sidebar_position: 100 --- The most common way to create a data asset in Dagster is by annotating a function with an asset decorator. The function computes the contents of the asset, such as a database table or file. @@ -71,6 +71,6 @@ flowchart LR ## Next steps -- Learn to create [dependencies between assets](/guides/asset-dependencies) -- Enrich Dagster's built-in data catalog with [asset metadata](/guides/metadata) -- Learn to use a [factory pattern](/guides/asset-factories) to create multiple, similar assets +- Learn to create [dependencies between assets](/guides/build/assets-concepts/asset-dependencies) +- Enrich Dagster's built-in data catalog with [asset metadata](/guides/build/create-a-pipeline/metadata) +- Learn to use a [factory pattern](/guides/build/configure/asset-factories) to create multiple, similar assets diff --git a/docs/docs-beta/docs/guides/external-assets.md b/docs/docs-beta/docs/guides/build/create-a-pipeline/external-assets.md similarity index 91% rename from docs/docs-beta/docs/guides/external-assets.md rename to docs/docs-beta/docs/guides/build/create-a-pipeline/external-assets.md index 05f2ae22d720c..482539981cc84 100644 --- a/docs/docs-beta/docs/guides/external-assets.md +++ b/docs/docs-beta/docs/guides/build/create-a-pipeline/external-assets.md @@ -1,6 +1,6 @@ --- title: Representing external data sources with external assets -sidebar_position: 80 +sidebar_position: 500 sidebar_label: 'External data sources' --- @@ -21,8 +21,8 @@ For example, external assets could be: To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/guides/data-assets) -- Familiarity with [Sensors](/guides/sensors) +- Familiarity with [Assets](/guides/build/create-a-pipeline/data-assets) +- Familiarity with [Sensors](/guides/automate/sensors) ## Defining external assets @@ -46,13 +46,13 @@ There are two main ways to do this: ### Pulling with sensors -You can use a Dagster [sensor](/guides/sensors) to regularly poll the external system and pull information about the external asset into Dagster. +You can use a Dagster [sensor](/guides/automate/sensors) to regularly poll the external system and pull information about the external asset into Dagster. For example, here's how you would poll an external system like an SFTP server to update an external asset whenever the file is changed. -Refer to the [Sensors guide](/guides/sensors) for more information about sensors. +Refer to the [Sensors guide](/guides/automate/sensors) for more information about sensors. ### Pushing with the REST API diff --git a/docs/docs-beta/docs/guides/pipelines.md b/docs/docs-beta/docs/guides/build/create-a-pipeline/index.md similarity index 84% rename from docs/docs-beta/docs/guides/pipelines.md rename to docs/docs-beta/docs/guides/build/create-a-pipeline/index.md index 6609d18797e0c..db67cd5184f1d 100644 --- a/docs/docs-beta/docs/guides/pipelines.md +++ b/docs/docs-beta/docs/guides/build/create-a-pipeline/index.md @@ -1,6 +1,7 @@ --- title: "Create a pipeline" description: "Learn how to create data pipelines using Dagster's asset-based approach" +sidebar_position: 10 --- In Dagster, data pipelines are created using an asset-based approach. This overview will introduce you to the key concepts and steps involved in building a Dagster pipeline. @@ -45,10 +46,10 @@ Most Dagster pipelines follow these steps: To start building your Dagster pipeline, dive into the following guides: -- [Define data assets](/guides/data-assets) -- [Create dependencies between assets](/guides/asset-dependencies) -- [Enrich assets with metadata](/guides/metadata) -- [Partition assets](/guides/partitioning) -- [Represent external data sources](/guides/external-assets) +- [Define data assets](/guides/build/create-a-pipeline/data-assets) +- [Create dependencies between assets](/guides/build/assets-concepts/asset-dependencies) +- [Enrich assets with metadata](/guides/build/create-a-pipeline/metadata) +- [Partition assets](/guides/build/create-a-pipeline/partitioning) +- [Represent external data sources](/guides/build/create-a-pipeline/external-assets) By following these guides, you'll learn how to create powerful, maintainable data pipelines using Dagster's asset-based approach. diff --git a/docs/docs-beta/docs/guides/metadata.md b/docs/docs-beta/docs/guides/build/create-a-pipeline/metadata.md similarity index 96% rename from docs/docs-beta/docs/guides/metadata.md rename to docs/docs-beta/docs/guides/build/create-a-pipeline/metadata.md index ae668e3cd0076..a271bb38f6335 100644 --- a/docs/docs-beta/docs/guides/metadata.md +++ b/docs/docs-beta/docs/guides/build/create-a-pipeline/metadata.md @@ -1,7 +1,7 @@ --- title: 'Adding tags and metadata to assets' description: 'Learn how to add tags and metadata to assets to improve observability in Dagster' -sidebar_position: 40 +sidebar_position: 200 sidebar_label: 'Add metadata' --- @@ -19,7 +19,7 @@ Using metadata in Dagster, you can: To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/guides/data-assets) +- Familiarity with [Assets](/guides/build/create-a-pipeline/data-assets) ## Adding owners to assets \{#owners} @@ -31,7 +31,7 @@ In a large organization, it's important to know which individuals and teams are `owners` must either be an email address or a team name prefixed by `team:`. :::tip -With Dagster+ Pro, you can create asset-based alerts that automatically notify an asset's owners when triggered. Refer to the [Dagster+ alert documentation](/dagster-plus/deployment/alerts) for more information. +With Dagster+ Pro, you can create asset-based alerts that automatically notify an asset's owners when triggered. Refer to the [Dagster+ alert documentation](/dagster-plus/features/alerts) for more information. ::: ## Organizing assets with tags \{#tags} @@ -116,7 +116,7 @@ Dagster+ provides rich visualization and navigation of column lineage in the Ass ## Linking assets with source code \{#source-code} -import Experimental from '../partials/\_Experimental.md'; +import Experimental from '../../../partials/\_Experimental.md'; @@ -134,7 +134,7 @@ Dagster can automatically attach code references to assets during local developm ### Customizing code references \{#custom-references} -If you want to customize how code references are attached - such as when you are building [domain-specific languages with asset factories](/guides/asset-factories) - you can manually add the `dagster/code_references` metadata to asset definitions: +If you want to customize how code references are attached - such as when you are building [domain-specific languages with asset factories](/guides/build/configure/asset-factories) - you can manually add the `dagster/code_references` metadata to asset definitions: diff --git a/docs/docs-beta/docs/guides/partition-dependencies.md b/docs/docs-beta/docs/guides/build/create-a-pipeline/partition-dependencies.md similarity index 92% rename from docs/docs-beta/docs/guides/partition-dependencies.md rename to docs/docs-beta/docs/guides/build/create-a-pipeline/partition-dependencies.md index a42687b21b41b..d6ee59ba03478 100644 --- a/docs/docs-beta/docs/guides/partition-dependencies.md +++ b/docs/docs-beta/docs/guides/build/create-a-pipeline/partition-dependencies.md @@ -2,7 +2,7 @@ title: Defining dependencies between partitioned assets description: Learn how to define dependencies between partitioned and unpartitioned assets in Dagster. sidebar_label: Partition dependencies -sidebar_position: 31 +sidebar_position: 400 --- Now that you've seen how to model partitioned assets in different ways, you may want to define dependencies between the partitioned assets, or even between unpartitioned assets. @@ -19,8 +19,8 @@ Partitioned assets in Dagster can have dependencies on other partitioned assets, To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/guides/data-assets) -- Familiarity with [Partitions](/guides/partitioning) +- Familiarity with [Assets](/guides/build/create-a-pipeline/data-assets) +- Familiarity with [Partitions](/guides/build/create-a-pipeline/partitioning) @@ -53,7 +53,7 @@ In this example: - First, we specify `automation_condition=AutomationCondition.eager()` to the `weekly_sales_summary` asset. This ensures it runs weekly after all seven daily partitions of `daily_sales_data` are up-to-date. - Second, we specify `automation_condition=AutomationCondition.cron(cron_schedule="0 1 * * *")` to the `daily_sales_data` asset. This ensures it runs daily. -Note: In a simpler example above, we manually set up a daily schedule for asset execution. For more complex dependency logic, it's recommended to use automation conditions instead of schedules. Automation conditions specify when an asset should run, which allows you to define execution criteria without custom scheduling logic. For more details, see [Declarative Automation](/concepts/automation/declarative-automation). +Note: In a simpler example above, we manually set up a daily schedule for asset execution. For more complex dependency logic, it's recommended to use automation conditions instead of schedules. Automation conditions specify when an asset should run, which allows you to define execution criteria without custom scheduling logic. For more details, see [Declarative Automation](/guides/automate/declarative-automation). ## Dependencies between time-based partitions and un-partitioned assets diff --git a/docs/docs-beta/docs/guides/partitioning.md b/docs/docs-beta/docs/guides/build/create-a-pipeline/partitioning.md similarity index 95% rename from docs/docs-beta/docs/guides/partitioning.md rename to docs/docs-beta/docs/guides/build/create-a-pipeline/partitioning.md index 45f0a5c4c6c4b..e82e16a28c601 100644 --- a/docs/docs-beta/docs/guides/partitioning.md +++ b/docs/docs-beta/docs/guides/build/create-a-pipeline/partitioning.md @@ -2,7 +2,7 @@ title: Partitioning assets description: Learn how to partition your data in Dagster. sidebar_label: Partition data -sidebar_position: 30 +sidebar_position: 300 --- In Dagster, partitioning is a powerful technique for managing large datasets, improving pipeline performance, and enabling incremental processing. This guide will help you understand how to implement data partitioning in your Dagster projects. @@ -19,7 +19,7 @@ There are several ways to partition your data in Dagster: To follow the steps in this guide, you'll need: -- Familiarity with [Assets](/guides/data-assets) +- Familiarity with [Assets](/guides/build/create-a-pipeline/data-assets) @@ -58,7 +58,7 @@ Consider this example: -Because the partition values are unknown in advance, `DynamicPartitionsDefinition` is used to define the partition. Then, the `all_regions_sensor` +Because the partition values are unknown in advance, `DynamicPartitionsDefinition` is used to define the partition. Then, the `all_regions_sensor` TODO: incomplete sentence In this example: @@ -67,4 +67,4 @@ In this example: ## Next steps -- TODOD: Partition dependencies \ No newline at end of file +- TODO: Partition dependencies \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/build/index.md b/docs/docs-beta/docs/guides/build/index.md new file mode 100644 index 0000000000000..9551dba1946eb --- /dev/null +++ b/docs/docs-beta/docs/guides/build/index.md @@ -0,0 +1,7 @@ +--- +title: "Building pipelines" +sidebar_position: 30 +sidebar_class_name: hidden +--- + +This section is about building asset pipelines. \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/build-your-own.md b/docs/docs-beta/docs/guides/build/integrate/build-your-own.md similarity index 73% rename from docs/docs-beta/docs/guides/build-your-own.md rename to docs/docs-beta/docs/guides/build/integrate/build-your-own.md index c526e0ca0e8fd..6c5e3008a33e7 100644 --- a/docs/docs-beta/docs/guides/build-your-own.md +++ b/docs/docs-beta/docs/guides/build/integrate/build-your-own.md @@ -1,5 +1,5 @@ --- title: "Building an asset integration" -sidebar_position: 70 +sidebar_position: 400 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/cloud-services.md b/docs/docs-beta/docs/guides/build/integrate/cloud-services.md similarity index 72% rename from docs/docs-beta/docs/guides/cloud-services.md rename to docs/docs-beta/docs/guides/build/integrate/cloud-services.md index 04ccde2824574..1fdc39f3d75fd 100644 --- a/docs/docs-beta/docs/guides/cloud-services.md +++ b/docs/docs-beta/docs/guides/build/integrate/cloud-services.md @@ -1,5 +1,5 @@ --- title: Connecting to cloud services -sidebar_position: 25 +sidebar_position: 600 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/build/integrate/index.mdx b/docs/docs-beta/docs/guides/build/integrate/index.mdx new file mode 100644 index 0000000000000..3aa86ed1f629e --- /dev/null +++ b/docs/docs-beta/docs/guides/build/integrate/index.mdx @@ -0,0 +1,8 @@ +--- +title: "Integrate" +sidebar_position: 30 +--- + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/ingesting-data.md b/docs/docs-beta/docs/guides/build/integrate/ingesting-data.md similarity index 88% rename from docs/docs-beta/docs/guides/ingesting-data.md rename to docs/docs-beta/docs/guides/build/integrate/ingesting-data.md index a7786b94ffb78..a6290c8c37c1d 100644 --- a/docs/docs-beta/docs/guides/ingesting-data.md +++ b/docs/docs-beta/docs/guides/build/integrate/ingesting-data.md @@ -1,7 +1,7 @@ --- title: Ingesting data with Dagster description: Learn how to orchestrate data ingestion with Dagster -sidebar_position: 10 +sidebar_position: 100 sidebar_label: Ingest data --- @@ -18,7 +18,7 @@ Dagster is often used to orchestrate the ingestion of data into a data warehouse To follow this guide, you'll need: -- Familiarity with [Assets](/concepts/assets) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx ## How Dagster supports data ingestion @@ -48,5 +48,5 @@ For example, imagine there's a CSV file of counties on the internet and you want ## Next steps -- Transform data using [Dagster's dbt integration](/guides/transform-dbt) -- Use asset checks [to test data quality](/guides/asset-checks) and [freshness](/guides/data-freshness-testing) \ No newline at end of file +- Transform data using [Dagster's dbt integration](/guides/build/integrate/transform-dbt) +- Use asset checks [to test data quality](/guides/test/asset-checks) and [freshness](/guides/test/data-freshness-testing) \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/non-python.md b/docs/docs-beta/docs/guides/build/integrate/non-python.md similarity index 95% rename from docs/docs-beta/docs/guides/non-python.md rename to docs/docs-beta/docs/guides/build/integrate/non-python.md index 02e45eb0f01e5..1f2618b62e259 100644 --- a/docs/docs-beta/docs/guides/non-python.md +++ b/docs/docs-beta/docs/guides/build/integrate/non-python.md @@ -1,7 +1,7 @@ --- title: "Using Dagster Pipes to execute non-Python languages" sidebar_label: "Dagster Pipes" -sidebar_position: 60 +sidebar_position: 300 --- Dagster is written in Python, but that doesn't mean it's that Python is the only language that can be used when materializing assets. With Dagster Pipes, you can run code in other languages and send information back to Dagster. @@ -13,7 +13,7 @@ This guide covers how to run JavaScript with Dagster using Pipes, however, the s To follow this guide, you'll need: -- Familiarity with [Assets](/concepts/assets) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx - A basic understanding of JavaScript and Node.js To run the examples, you'll need to install: @@ -80,5 +80,5 @@ Finally, update your Dagster asset to pass in the model information that's used ## What's next? -- Schedule your pipeline to run periodically with [Automating Pipelines](/guides/automation) -- Explore adding asset checks to validate your script with [Understanding Asset Checks](/concepts/assets/asset-checks) +- Schedule your pipeline to run periodically with [Automating Pipelines](/guides/automate/index.md) +- Explore adding asset checks to validate your script with [Understanding Asset Checks](/guides/test/asset-checks) diff --git a/docs/docs-beta/docs/guides/pipes.md b/docs/docs-beta/docs/guides/build/integrate/pipes.md similarity index 96% rename from docs/docs-beta/docs/guides/pipes.md rename to docs/docs-beta/docs/guides/build/integrate/pipes.md index 2ded7841f8de5..39a6152f256f2 100644 --- a/docs/docs-beta/docs/guides/pipes.md +++ b/docs/docs-beta/docs/guides/build/integrate/pipes.md @@ -1,6 +1,6 @@ --- title: "Executing code outside of Dagster with Pipes" -sidebar_position: 40 +sidebar_position: 500 sidebar_label: "External execution with Pipes" --- @@ -13,7 +13,7 @@ In this guide, we'll walk you through how to invoke non-Dagster code through Pip
Prerequisites -- Familiarity with [Assets](/concepts/assets) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx
## Setting up an asset that invokes your external code diff --git a/docs/docs-beta/docs/guides/transform-dbt.md b/docs/docs-beta/docs/guides/build/integrate/transform-dbt.md similarity index 97% rename from docs/docs-beta/docs/guides/transform-dbt.md rename to docs/docs-beta/docs/guides/build/integrate/transform-dbt.md index 758133ae012e7..55dcf07232edb 100644 --- a/docs/docs-beta/docs/guides/transform-dbt.md +++ b/docs/docs-beta/docs/guides/build/integrate/transform-dbt.md @@ -1,10 +1,7 @@ --- title: Transforming data with dbt -sidebar_position: 20 +sidebar_position: 200 sidebar_label: Transform data with dbt -last_update: - date: 2024-08-26 - author: Nick Roach --- Dagster orchestrates dbt alongside other technologies, so you can schedule dbt with Spark, Python, etc. in a single data pipeline. Dagster's asset-oriented approach allows Dagster to understand dbt at the level of individual dbt models. diff --git a/docs/docs-beta/docs/guides/build/ops-jobs/index.md b/docs/docs-beta/docs/guides/build/ops-jobs/index.md new file mode 100644 index 0000000000000..c03c4a7beccb6 --- /dev/null +++ b/docs/docs-beta/docs/guides/build/ops-jobs/index.md @@ -0,0 +1,10 @@ +--- +unlisted: true +sidebar_position: 50 +--- + +# Ops and jobs + +import DocCardList from '@theme/DocCardList'; + + diff --git a/docs/docs-beta/docs/concepts/ops-jobs/job-configuration.md b/docs/docs-beta/docs/guides/build/ops-jobs/job-configuration.md similarity index 76% rename from docs/docs-beta/docs/concepts/ops-jobs/job-configuration.md rename to docs/docs-beta/docs/guides/build/ops-jobs/job-configuration.md index 23be86378acca..c15ec31bab52e 100644 --- a/docs/docs-beta/docs/concepts/ops-jobs/job-configuration.md +++ b/docs/docs-beta/docs/guides/build/ops-jobs/job-configuration.md @@ -1,6 +1,6 @@ --- title: "Job configuration" -sidebar_position: 10 +sidebar_position: 200 unlisted: true --- diff --git a/docs/docs-beta/docs/concepts/ops-jobs/ops-vs-assets.md b/docs/docs-beta/docs/guides/build/ops-jobs/ops-vs-assets.md similarity index 74% rename from docs/docs-beta/docs/concepts/ops-jobs/ops-vs-assets.md rename to docs/docs-beta/docs/guides/build/ops-jobs/ops-vs-assets.md index 81795aa23e439..d39a086dd74f8 100644 --- a/docs/docs-beta/docs/concepts/ops-jobs/ops-vs-assets.md +++ b/docs/docs-beta/docs/guides/build/ops-jobs/ops-vs-assets.md @@ -1,6 +1,6 @@ --- title: "Ops vs. assets" -sidebar_position: 10 +sidebar_position: 100 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/project-structure.md b/docs/docs-beta/docs/guides/build/project-structure.md similarity index 99% rename from docs/docs-beta/docs/guides/project-structure.md rename to docs/docs-beta/docs/guides/build/project-structure.md index bae009c702cd7..6a9d638d05856 100644 --- a/docs/docs-beta/docs/guides/project-structure.md +++ b/docs/docs-beta/docs/guides/build/project-structure.md @@ -1,5 +1,6 @@ --- title: "How to structure your Dagster project" +sidebar_position: 200 --- # How to structure your Dagster project diff --git a/docs/docs-beta/docs/guides/configuring.md b/docs/docs-beta/docs/guides/configuring.md deleted file mode 100644 index 30f910f45bf31..0000000000000 --- a/docs/docs-beta/docs/guides/configuring.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Configuring pipelines and runs -sidebar_label: Configuring pipelines -sidebar_position: 40 -unlisted: true ---- diff --git a/docs/docs-beta/docs/guides/dagster-plus.md b/docs/docs-beta/docs/guides/dagster-plus.md deleted file mode 100644 index bc65f96ffff0b..0000000000000 --- a/docs/docs-beta/docs/guides/dagster-plus.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: "Deploy to Dagster+" -sidebar_position: 40 ---- - -# Deploy to Dagster+ - -See the [Dagster+ getting started](/dagster-plus/getting-started) guide. diff --git a/docs/docs-beta/docs/guides/deploy/code-locations/index.mdx b/docs/docs-beta/docs/guides/deploy/code-locations/index.mdx new file mode 100644 index 0000000000000..7951c9ba3dbdf --- /dev/null +++ b/docs/docs-beta/docs/guides/deploy/code-locations/index.mdx @@ -0,0 +1,8 @@ +--- +title: Code locations +sidebar_position: 30 +--- + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/code-locations.md b/docs/docs-beta/docs/guides/deploy/code-locations/managing-code-locations.md similarity index 58% rename from docs/docs-beta/docs/guides/code-locations.md rename to docs/docs-beta/docs/guides/deploy/code-locations/managing-code-locations.md index 7316b34d8b440..b44a57a85cad4 100644 --- a/docs/docs-beta/docs/guides/code-locations.md +++ b/docs/docs-beta/docs/guides/deploy/code-locations/managing-code-locations.md @@ -1,7 +1,5 @@ --- title: "Managing code locations with Definitions" -sidebar_position: 50 -sidebar_label: "Code locations and Definitions" unlisted: true --- diff --git a/docs/docs-beta/docs/reference/workspace-yaml.md b/docs/docs-beta/docs/guides/deploy/code-locations/workspace-yaml.md similarity index 94% rename from docs/docs-beta/docs/reference/workspace-yaml.md rename to docs/docs-beta/docs/guides/deploy/code-locations/workspace-yaml.md index e2736189bb9ee..4ee95d7f1a0eb 100644 --- a/docs/docs-beta/docs/reference/workspace-yaml.md +++ b/docs/docs-beta/docs/guides/deploy/code-locations/workspace-yaml.md @@ -1,7 +1,11 @@ -# Workspace File Reference +--- +title: "workspace.yaml reference" +--- + +# Workspace file reference :::info - This reference is only applicable to Dagster OSS, for Dagster Cloud see [the Dagster Cloud Code Locations guide](/dagster-plus/deployment/code-locations) + This reference is only applicable to Dagster OSS. For Dagster Cloud see [the Dagster Cloud Code Locations guide](/dagster-plus/deployment/code-locations) ::: The `workspace.yaml` file is used to configure code locations in Dagster. It tells Dagster where to find your code and how to load it. @@ -16,7 +20,7 @@ To load the workspace.yaml file from a different folder, use the -w argument: dagster dev -w path/to/workspace.yaml ``` -## File Structure +## File structure The `workspace.yaml` file uses the following structure: @@ -33,7 +37,7 @@ Where `` can be one of: -## Loading Methods +## Loading methods We recommend using `python_module` for most use cases. @@ -93,7 +97,7 @@ load_from: location_name: "my_grpc_server" ``` -## Multiple Code Locations +## Multiple code locations You can define multiple code locations in a single `workspace.yaml` file: diff --git a/docs/docs-beta/docs/reference/dagster-yaml.md b/docs/docs-beta/docs/guides/deploy/dagster-yaml.md similarity index 97% rename from docs/docs-beta/docs/reference/dagster-yaml.md rename to docs/docs-beta/docs/guides/deploy/dagster-yaml.md index 9b96aed2ebbaf..d7f628b9f533a 100644 --- a/docs/docs-beta/docs/reference/dagster-yaml.md +++ b/docs/docs-beta/docs/guides/deploy/dagster-yaml.md @@ -1,12 +1,17 @@ -# dagster.yaml Reference +--- +title: "dagster.yaml reference" +sidebar_position: 200 +---- + +# dagster.yaml reference The `dagster.yaml` file is used to configure the Dagster instance. It defines various settings for storage, run execution, logging, and other aspects of a Dagster deployment. -## File Location +## File location By default, Dagster looks for the `dagster.yaml` file in the directory specified by the `DAGSTER_HOME` environment variable. -## Using Environment Variables +## Using environment variables You can use environment variables to override values in the `dagster.yaml` file. @@ -19,7 +24,7 @@ instance: env: ME_ENV_VAR ``` -## Full Configuration Specification +## Full configuration specification Here's a comprehensive `dagster.yaml` specification with all available options: @@ -159,7 +164,7 @@ auto_materialize: num_workers: 4 ``` -## Configuration Options +## Configuration options ### `storage` diff --git a/docs/docs-beta/docs/guides/amazon-web-services.md b/docs/docs-beta/docs/guides/deploy/deployment-options/amazon-web-services.md similarity index 73% rename from docs/docs-beta/docs/guides/amazon-web-services.md rename to docs/docs-beta/docs/guides/deploy/deployment-options/amazon-web-services.md index 74f8b5db5e704..d425e912d4cd3 100644 --- a/docs/docs-beta/docs/guides/amazon-web-services.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/amazon-web-services.md @@ -1,5 +1,5 @@ --- title: "Deploy to Amazon Web Services" -sidebar_position: 10 unlisted: true +sidebar_position: 500 --- diff --git a/docs/docs-beta/docs/guides/dagster-service.md b/docs/docs-beta/docs/guides/deploy/deployment-options/dagster-service.md similarity index 97% rename from docs/docs-beta/docs/guides/dagster-service.md rename to docs/docs-beta/docs/guides/deploy/deployment-options/dagster-service.md index 37f9a27285f74..fcb65fafa58f9 100644 --- a/docs/docs-beta/docs/guides/dagster-service.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/dagster-service.md @@ -1,6 +1,7 @@ --- title: 'Deploy Dagster as a service' description: 'Learn how to deploy Dagster as a service on a single machine' +sidebar_position: 200 ---
@@ -114,4 +115,4 @@ A deployment can have multiple instances of `dagster-webserver`, but should incl Now that you have Dagster running as a service, you might want to explore: - [Configuring your Dagster instance](/todo) -- [Setting up schedules and sensors](/guides/schedules) +- [Setting up schedules and sensors](/guides/automate/schedules) diff --git a/docs/docs-beta/docs/guides/docker.md b/docs/docs-beta/docs/guides/deploy/deployment-options/docker.md similarity index 99% rename from docs/docs-beta/docs/guides/docker.md rename to docs/docs-beta/docs/guides/deploy/deployment-options/docker.md index d2be589f3fca6..4247a956771c8 100644 --- a/docs/docs-beta/docs/guides/docker.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/docker.md @@ -1,6 +1,7 @@ --- title: "Deploying with Docker Compose" description: A guide to deploying Dagster with Docker Compose. +sidebar_position: 400 --- This guide provides instructions for deploying Dagster using Docker Compose. This is useful when you want to, for example, deploy Dagster on an AWS EC2 host. A typical Dagster Docker deployment includes a several long-running containers: one for the webserver, one for the daemon, and one for each code location. It also typically executes each run in its own container. diff --git a/docs/docs-beta/docs/guides/google-cloud-platform.md b/docs/docs-beta/docs/guides/deploy/deployment-options/google-cloud-platform.md similarity index 74% rename from docs/docs-beta/docs/guides/google-cloud-platform.md rename to docs/docs-beta/docs/guides/deploy/deployment-options/google-cloud-platform.md index 514c0695254c3..0cdd2376cb6b5 100644 --- a/docs/docs-beta/docs/guides/google-cloud-platform.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/google-cloud-platform.md @@ -1,5 +1,5 @@ --- title: "Deploy to Google Cloud Platform" -sidebar_position: 20 unlisted: true +sidebar_position: 600 --- diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/index.mdx b/docs/docs-beta/docs/guides/deploy/deployment-options/index.mdx new file mode 100644 index 0000000000000..757020a9b3852 --- /dev/null +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/index.mdx @@ -0,0 +1,8 @@ +--- +title: "Deployment options" +sidebar_position: 10 +--- + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/kubernetes.md b/docs/docs-beta/docs/guides/deploy/deployment-options/kubernetes.md similarity index 99% rename from docs/docs-beta/docs/guides/kubernetes.md rename to docs/docs-beta/docs/guides/deploy/deployment-options/kubernetes.md index 1f5c15b648da6..f29729c61072b 100644 --- a/docs/docs-beta/docs/guides/kubernetes.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/kubernetes.md @@ -1,6 +1,6 @@ --- title: "Deploy to Kubernetes" -sidebar_position: 21 +sidebar_position: 300 --- This guide will walk you through how to run the Dagster-specific components of a Dagster production deployment on a Kubernetes cluster. This includes the Dagster daemon, a webserver to serve the Dagster UI, a PostgrSQL container, and your Dagster project user code. diff --git a/docs/docs-beta/docs/guides/microsoft-azure.md b/docs/docs-beta/docs/guides/deploy/deployment-options/microsoft-azure.md similarity index 72% rename from docs/docs-beta/docs/guides/microsoft-azure.md rename to docs/docs-beta/docs/guides/deploy/deployment-options/microsoft-azure.md index d26b6d4f5f4f7..a33096ceb12c0 100644 --- a/docs/docs-beta/docs/guides/microsoft-azure.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/microsoft-azure.md @@ -1,5 +1,5 @@ --- title: "Deploy to Microsoft Azure" -sidebar_position: 30 unlisted: true +sidebar_position: 700 --- diff --git a/docs/docs-beta/docs/guides/running-local-ui-development.md b/docs/docs-beta/docs/guides/deploy/deployment-options/running-local-ui-development.md similarity index 95% rename from docs/docs-beta/docs/guides/running-local-ui-development.md rename to docs/docs-beta/docs/guides/deploy/deployment-options/running-local-ui-development.md index d65775c6bfc79..f6e84c3fafe95 100644 --- a/docs/docs-beta/docs/guides/running-local-ui-development.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/running-local-ui-development.md @@ -1,6 +1,7 @@ --- title: Running Dagster locally description: How to run Dagster on your local machine. +sidebar_position: 100 --- # Running Dagster locally @@ -8,7 +9,7 @@ description: How to run Dagster on your local machine. In this guide, we'll walk you through how to run Dagster on your local machine using the `dagster dev` command. The `dagster dev` command launches the Dagster UI and the Dagster daemon, allowing you to start a full deployment of Dagster from the command line. :::warning -`dagster dev` is intended for local development _only_. If you want to run Dagster for production use cases, see our [Deployment](/guides/deployment) guides. +`dagster dev` is intended for local development _only_. If you want to run Dagster for production use cases, see our other [deployment guides](/guides/deploy/deployment-options/index.mdx). ::: ## Locating your code diff --git a/docs/docs-beta/docs/concepts/execution/dagster-daemon.md b/docs/docs-beta/docs/guides/deploy/execution/dagster-daemon.md similarity index 74% rename from docs/docs-beta/docs/concepts/execution/dagster-daemon.md rename to docs/docs-beta/docs/guides/deploy/execution/dagster-daemon.md index e19cbb520dbeb..85da08fe450d3 100644 --- a/docs/docs-beta/docs/concepts/execution/dagster-daemon.md +++ b/docs/docs-beta/docs/guides/deploy/execution/dagster-daemon.md @@ -1,6 +1,6 @@ --- title: "Dagster daemon" -sidebar_position: 10 +sidebar_position: 100 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deploy/execution/index.mdx b/docs/docs-beta/docs/guides/deploy/execution/index.mdx new file mode 100644 index 0000000000000..c1bbbc2b8a5e0 --- /dev/null +++ b/docs/docs-beta/docs/guides/deploy/execution/index.mdx @@ -0,0 +1,8 @@ +--- +title: "Execution" +sidebar_position: 20 +--- + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/concepts/execution/run-coordinators.md b/docs/docs-beta/docs/guides/deploy/execution/run-coordinators.md similarity index 75% rename from docs/docs-beta/docs/concepts/execution/run-coordinators.md rename to docs/docs-beta/docs/guides/deploy/execution/run-coordinators.md index aa871f1e371ed..48ce80b1690cc 100644 --- a/docs/docs-beta/docs/concepts/execution/run-coordinators.md +++ b/docs/docs-beta/docs/guides/deploy/execution/run-coordinators.md @@ -1,6 +1,6 @@ --- title: "Run coordinators" -sidebar_position: 40 +sidebar_position: 400 unlisted: true --- diff --git a/docs/docs-beta/docs/concepts/execution/run-executors.md b/docs/docs-beta/docs/guides/deploy/execution/run-executors.md similarity index 74% rename from docs/docs-beta/docs/concepts/execution/run-executors.md rename to docs/docs-beta/docs/guides/deploy/execution/run-executors.md index ff006420ac983..7351a18aefd9d 100644 --- a/docs/docs-beta/docs/concepts/execution/run-executors.md +++ b/docs/docs-beta/docs/guides/deploy/execution/run-executors.md @@ -1,6 +1,6 @@ --- title: "Run executors" -sidebar_position: 30 +sidebar_position: 300 unlisted: true --- diff --git a/docs/docs-beta/docs/concepts/execution/run-launchers.md b/docs/docs-beta/docs/guides/deploy/execution/run-launchers.md similarity index 74% rename from docs/docs-beta/docs/concepts/execution/run-launchers.md rename to docs/docs-beta/docs/guides/deploy/execution/run-launchers.md index f08630d093ebb..f4982c170a6b3 100644 --- a/docs/docs-beta/docs/concepts/execution/run-launchers.md +++ b/docs/docs-beta/docs/guides/deploy/execution/run-launchers.md @@ -1,6 +1,6 @@ --- title: "Run launchers" -sidebar_position: 20 +sidebar_position: 200 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deploy/index.md b/docs/docs-beta/docs/guides/deploy/index.md new file mode 100644 index 0000000000000..3a5ec3fb353d1 --- /dev/null +++ b/docs/docs-beta/docs/guides/deploy/index.md @@ -0,0 +1,15 @@ +--- +title: "Deploying Dagster" +sidebar_position: 70 +sidebar_class_name: hidden +--- + +# Deploying Dagster + +This section is about self-hosting Dagster. + +:::info +To deploy to Dagster+, see the [Dagster+ getting started guide](/dagster-plus/getting-started). + +To migrate from self-hosted to Dagster+, see [Migrate from self-hosted to Dagster+](/dagster-plus/deployment/migration/self-hosted-to-dagster-plus). +::: diff --git a/docs/docs-beta/docs/guides/secrets.md b/docs/docs-beta/docs/guides/deploy/secrets.md similarity index 68% rename from docs/docs-beta/docs/guides/secrets.md rename to docs/docs-beta/docs/guides/deploy/secrets.md index d26e2a3b32571..5134eb92e17aa 100644 --- a/docs/docs-beta/docs/guides/secrets.md +++ b/docs/docs-beta/docs/guides/deploy/secrets.md @@ -1,5 +1,5 @@ --- title: Managing secrets -sidebar_position: 50 +sidebar_position: 100 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deployment.md b/docs/docs-beta/docs/guides/deployment.md deleted file mode 100644 index 88b03ec02f293..0000000000000 --- a/docs/docs-beta/docs/guides/deployment.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Deployment" -unlisted: true ---- - -# Deployment diff --git a/docs/docs-beta/docs/guides/alerting.md b/docs/docs-beta/docs/guides/monitor/alerting.md similarity index 83% rename from docs/docs-beta/docs/guides/alerting.md rename to docs/docs-beta/docs/guides/monitor/alerting.md index 05d4545e02012..92c409cce7b8a 100644 --- a/docs/docs-beta/docs/guides/alerting.md +++ b/docs/docs-beta/docs/guides/monitor/alerting.md @@ -1,13 +1,13 @@ --- title: "Setting up alerts" -sidebar_position: 30 -sidebar_label: "Alerting" +sidebar_position: 10 unlisted: true --- +(This will likely be a category) + Alerting if my pipeline didn't execute Tracking when a run or sensor fails Knowing when a pipeline never ran Knowing if a pipeline is running slow, or an asset is late Knowing if my Dagster instance is having issues - diff --git a/docs/docs-beta/docs/guides/monitor/index.md b/docs/docs-beta/docs/guides/monitor/index.md new file mode 100644 index 0000000000000..aaf06312205dc --- /dev/null +++ b/docs/docs-beta/docs/guides/monitor/index.md @@ -0,0 +1,7 @@ +--- +title: "Monitoring pipelines" +sidebar_position: 50 +sidebar_class_name: hidden +--- + +This section is about monitoring pipelines. \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/custom-logging.md b/docs/docs-beta/docs/guides/monitor/logging/custom-logging.md similarity index 99% rename from docs/docs-beta/docs/guides/custom-logging.md rename to docs/docs-beta/docs/guides/monitor/logging/custom-logging.md index 79fe115a23cb0..c3bf0380c5621 100644 --- a/docs/docs-beta/docs/guides/custom-logging.md +++ b/docs/docs-beta/docs/guides/monitor/logging/custom-logging.md @@ -1,6 +1,6 @@ --- title: "Setting up custom logging" -sidebar_position: 20 +sidebar_position: 100 --- # Custom loggers diff --git a/docs/docs-beta/docs/guides/custom-metrics-logs.md b/docs/docs-beta/docs/guides/monitor/logging/custom-metrics-logs.md similarity index 72% rename from docs/docs-beta/docs/guides/custom-metrics-logs.md rename to docs/docs-beta/docs/guides/monitor/logging/custom-metrics-logs.md index 06dc0a2a9cf97..62476124e059c 100644 --- a/docs/docs-beta/docs/guides/custom-metrics-logs.md +++ b/docs/docs-beta/docs/guides/monitor/logging/custom-metrics-logs.md @@ -1,5 +1,5 @@ --- title: "Use custom metrics in logs" -sidebar_position: 30 +sidebar_position: 200 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/monitor/logging/index.mdx b/docs/docs-beta/docs/guides/monitor/logging/index.mdx new file mode 100644 index 0000000000000..8de645e227aec --- /dev/null +++ b/docs/docs-beta/docs/guides/monitor/logging/index.mdx @@ -0,0 +1,8 @@ +--- +title: "Logging" +sidebar_position: 20 +--- + +import DocCardList from '@theme/DocCardList'; + + \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/some-guide.md b/docs/docs-beta/docs/guides/some-guide.md deleted file mode 100644 index 25867d5cbee78..0000000000000 --- a/docs/docs-beta/docs/guides/some-guide.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -unlisted: true ---- diff --git a/docs/docs-beta/docs/guides/asset-checks.md b/docs/docs-beta/docs/guides/test/asset-checks.md similarity index 79% rename from docs/docs-beta/docs/guides/asset-checks.md rename to docs/docs-beta/docs/guides/test/asset-checks.md index 3ee1edb59166c..3fddf035591b1 100644 --- a/docs/docs-beta/docs/guides/asset-checks.md +++ b/docs/docs-beta/docs/guides/test/asset-checks.md @@ -1,6 +1,6 @@ --- title: "Testing assets with Asset checks" -sidebar_position: 10 +sidebar_position: 100 sidebar_label: "Asset checks" --- @@ -17,7 +17,7 @@ Each asset check should test only a single asset property to keep tests uncompli To follow this guide, you'll need: -- Familiarity with [Assets](/concepts/assets) +- Familiarity with [Assets](/guides/build/assets-concepts/index.mdx
## Getting started @@ -33,14 +33,14 @@ To get started with asset checks, follow these general steps: ## Defining a single asset check \{#single-check} :::tip -Dagster's dbt integration can model existing dbt tests as asset checks. Refer to the [dagster-dbt documentaiton](/integrations/dbt) for more information. +Dagster's dbt integration can model existing dbt tests as asset checks. Refer to the [dagster-dbt documentation](/integrations/dbt) for more information. ::: A asset check is defined using the `@asset_check` decorator. The following example defines an asset check on an asset that fails if the `order_id` column of the asset contains a null value. The asset check will run after the asset has been materialized. - + ## Defining multiple asset checks \{#multiple-checks} @@ -53,13 +53,13 @@ The following example defines two asset checks using the `@multi_asset_check` de In this example, both asset checks will run in a single operation after the asset has been materialized. - + ## Programmatically generating asset checks \{#factory-pattern} Defining multiple checks can also be done using a factory pattern. The example below defines the same two asset checks as in the previous example, but this time using a factory pattern and the `@multi_asset_check` decorator. - + ## Blocking downstream materialization @@ -67,7 +67,7 @@ By default, if a parent's asset check fails during a run, the run will continue In the example bellow, if the `orders_id_has_no_nulls` check fails, the downstream `augmented_orders` asset won't be materialized. - + ## Scheduling and monitoring asset checks @@ -75,10 +75,10 @@ In some cases, running asset checks separately from the job materializing the as In the example below, two jobs are defined: one for the asset and another for the asset check. Schedules are defined to materialize the asset and execute the asset check independently. A sensor is defined to send an email alert when the asset check job fails. - + ## Next steps -- Learn more about assets in [Understanding Assets](/concepts/assets) -- Learn more about asset checks in [Understanding Asset Checks](/concepts/assets/asset-checks) +- Learn more about assets in [Understanding Assets](/guides/build/assets-concepts/index.mdx +- Learn more about asset checks in [Understanding Asset Checks](/guides/test/asset-checks) - Learn how to use [Great Expectations with Dagster](https://dagster.io/blog/ensuring-data-quality-with-dagster-and-great-expectations) diff --git a/docs/docs-beta/docs/guides/data-freshness-testing.md b/docs/docs-beta/docs/guides/test/data-freshness-testing.md similarity index 89% rename from docs/docs-beta/docs/guides/data-freshness-testing.md rename to docs/docs-beta/docs/guides/test/data-freshness-testing.md index dbd5c5a97130d..a738941c2d8f5 100644 --- a/docs/docs-beta/docs/guides/data-freshness-testing.md +++ b/docs/docs-beta/docs/guides/test/data-freshness-testing.md @@ -1,6 +1,6 @@ --- title: "Checking for data freshness" -sidebar_position: 20 +sidebar_position: 200 sidebar_label: "Data freshness" --- @@ -18,9 +18,9 @@ Freshness checks can also communicate SLAs for their data freshness. For example To follow the steps in this guide, you'll need familiarity with: -- [Assets](/guides/data-assets) -- [External assets](/guides/external-assets) -- [Asset checks](/guides/asset-checks) +- [Assets](/guides/build/create-a-pipeline/data-assets) +- [External assets](/guides/build/create-a-pipeline/external-assets) +- [Asset checks](/guides/test/asset-checks) @@ -45,7 +45,7 @@ The example below defines a freshness check on an asset that fails if the asset' ## External asset freshness \{#external-assets} -[External assets](/guides/external-assets) are assets orchestrated by systems other than Dagster. +[External assets](/guides/build/create-a-pipeline/external-assets) are assets orchestrated by systems other than Dagster. To run freshness checks on external assets, the checks need to know when the external assets were last updated. Emitting these update timestamps as values for the `dagster/last_updated_timestamp` observation metadata key allows Dagster to calculate whether the asset is overdue. @@ -73,9 +73,9 @@ If the asset hasn't been updated enough times, the check will pass with a messag Freshness check alerts are a Dagster+ feature. ::: -In Dagster+, you can set up alerts to notify you when assets are overdue for an update. Refer to the [Dagster+ alerting guide](/dagster-plus/deployment/alerts) for more information. +In Dagster+, you can set up alerts to notify you when assets are overdue for an update. Refer to the [Dagster+ alerting guide](/dagster-plus/features/alerts) for more information. ## Next steps -- Explore more [asset checks](/guides/asset-checks) -- Explore how to [raise alerts when assets are overdue](/dagster-plus/deployment/alerts) (Dagster+ Pro) \ No newline at end of file +- Explore more [asset checks](/guides/test/asset-checks) +- Explore how to [raise alerts when assets are overdue](/dagster-plus/features/alerts) (Dagster+ Pro) \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/test/index.md b/docs/docs-beta/docs/guides/test/index.md new file mode 100644 index 0000000000000..6284fc0b252bc --- /dev/null +++ b/docs/docs-beta/docs/guides/test/index.md @@ -0,0 +1,7 @@ +--- +title: "Testing assets" +sidebar_position: 60 +sidebar_class_name: hidden +--- + +This section is about testing assets. \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/unit-tests-assets-and-ops.md b/docs/docs-beta/docs/guides/test/unit-tests-assets-and-ops.md similarity index 92% rename from docs/docs-beta/docs/guides/unit-tests-assets-and-ops.md rename to docs/docs-beta/docs/guides/test/unit-tests-assets-and-ops.md index e98707e995760..057ef145945bd 100644 --- a/docs/docs-beta/docs/guides/unit-tests-assets-and-ops.md +++ b/docs/docs-beta/docs/guides/test/unit-tests-assets-and-ops.md @@ -1,6 +1,6 @@ --- title: "Unit testing assets and ops" -sidebar_position: 30 +sidebar_position: 300 sidebar_label: "Unit testing" --- @@ -15,8 +15,8 @@ This guide covers how to write unit tests for assets and ops with a variety of d To follow the steps in this guide, you'll need familiarity with: -- [Assets](/concepts/assets) -- [Ops and Jobs](/concepts/ops-jobs) +- [Assets](/guides/build/assets-concepts/index.mdx +- [Ops and Jobs](/guides/build/ops-jobs) ## Before you start @@ -106,6 +106,6 @@ If an asset or op has multiple parameters, it's recommended to use keyword argum ## Next steps -- Learn more about assets in [Understanding Assets](/concepts/assets) -- Learn more about ops in [Understanding Ops](/concepts/ops-jobs) -- Learn more about resources in [Resources](/concepts/resources) \ No newline at end of file +- Learn more about assets in [Understanding Assets](/guides/build/assets-concepts/index.mdx +- Learn more about ops in [Understanding Ops](/guides/build/ops-jobs) +- Learn more about resources in [Resources](/guides/build/configure/resources) \ No newline at end of file diff --git a/docs/docs-beta/docs/intro.md b/docs/docs-beta/docs/intro.md index c981a60dc34ef..56b22d535a3c7 100644 --- a/docs/docs-beta/docs/intro.md +++ b/docs/docs-beta/docs/intro.md @@ -29,10 +29,10 @@ Dagster is a data orchestrator built for data engineers, with integrated lineage Build your first Dagster pipeline in our Quickstart tutorial. - + New to Dagster? Learn about how thinking in assets can help you manage your data better. - + Learn about Dagster Plus, our managed offering that includes a hosted Dagster instance and many more features. diff --git a/docs/docs-beta/docs/reference/api-lifecycle.md b/docs/docs-beta/docs/reference/api-lifecycle.md index 22295ae299344..5b91b22fe3d39 100644 --- a/docs/docs-beta/docs/reference/api-lifecycle.md +++ b/docs/docs-beta/docs/reference/api-lifecycle.md @@ -1,5 +1,5 @@ --- -title: API Lifecycle Stages +title: API lifecycle stages --- This reference guide outlines the different stages in the lifecycle of Dagster APIs, from preview to deprecation. Understanding these stages helps developers make informed decisions about which APIs to use in their projects. @@ -8,7 +8,7 @@ Dagster's API lifecycle is designed to balance innovation with stability, ensuri The lifecycle stages described below provide a clear framework for understanding the maturity and reliability of different Dagster APIs. This transparency helps developers make informed decisions about which APIs to adopt in their projects, based on their specific needs for stability, feature completeness, and long-term support. -## API Lifecycle Stages +## API lifecycle stages | Stage | Description | Lifetime | |---------|------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| @@ -19,7 +19,7 @@ The lifecycle stages described below provide a clear framework for understanding | Deprecated | API is still available but will be removed in the future; avoid new usage. | Will be removed in a minor release, the DeprecationWarning will indicate the next release that will remove the API. -## Understanding the Stages +## Understanding the stages ### Preview - **Purpose**: For early testing and feedback diff --git a/docs/docs-beta/docs/reference/index.md b/docs/docs-beta/docs/reference/index.md deleted file mode 100644 index 1337f4d632a26..0000000000000 --- a/docs/docs-beta/docs/reference/index.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -title: References ---- \ No newline at end of file diff --git a/docs/docs-beta/docusaurus.config.ts b/docs/docs-beta/docusaurus.config.ts index 7403ab5e1f494..491dc86f6ba46 100644 --- a/docs/docs-beta/docusaurus.config.ts +++ b/docs/docs-beta/docusaurus.config.ts @@ -90,21 +90,15 @@ const config: Config = { { label: 'Dagster+', type: 'doc', - docId: 'dagster-plus', + docId: 'dagster-plus/index', position: 'left', }, { - label: 'API Docs', + label: 'API reference', type: 'doc', docId: 'api/index', position: 'left', }, - { - label: 'Reference', - type: 'doc', - docId: 'reference/index', - position: 'left', - }, //{ // label: 'Changelog', // type: 'doc', diff --git a/docs/docs-beta/scripts/vercel-sync-api-docs.sh b/docs/docs-beta/scripts/vercel-sync-api-docs.sh index fc4240432665d..d3ea8242400bd 100644 --- a/docs/docs-beta/scripts/vercel-sync-api-docs.sh +++ b/docs/docs-beta/scripts/vercel-sync-api-docs.sh @@ -13,7 +13,7 @@ cd .. export LC_ALL=C.UTF-8 curl -LsSf https://astral.sh/uv/install.sh | sh -source ~/.cargo/env +source "$HOME/.local/bin/env" uv python install 3.11 uv venv diff --git a/docs/docs-beta/sidebars.ts b/docs/docs-beta/sidebars.ts index a484606be4cbe..a5dc4f6105bb5 100644 --- a/docs/docs-beta/sidebars.ts +++ b/docs/docs-beta/sidebars.ts @@ -5,7 +5,7 @@ const sidebars: SidebarsConfig = { type: 'category', label: 'Getting Started', collapsed: false, - items: ['intro', 'getting-started/quickstart', 'getting-started/installation'], + items: ['intro', 'getting-started/quickstart', 'getting-started/installation', 'getting-started/glossary'], }, { type: 'category', @@ -17,152 +17,62 @@ const sidebars: SidebarsConfig = { type: 'category', label: 'Build', collapsed: false, + link: {type: 'doc', id: 'guides/build/index'}, items: [ { - type: 'category', - label: 'Create a pipeline', - link: {type: 'doc', id: 'guides/pipelines'}, - items: [ - 'guides/data-assets', - 'guides/metadata', - 'guides/partitioning', - 'guides/partition-dependencies', - 'guides/external-assets', - ], - }, - { - type: 'category', - label: 'Configure', - items: [ - 'guides/configuring-assets', - 'guides/asset-factories', - //'guides/resources', - 'guides/databases', - 'guides/apis', - 'guides/io-managers', - 'guides/managing-concurrency', - ], - }, - { - type: 'category', - label: 'Integrate', - items: ['guides/ingesting-data', 'guides/transform-dbt', 'guides/non-python'], - }, + type: 'autogenerated', + dirName: 'guides/build' + } ], }, { type: 'category', label: 'Automate', collapsed: false, + link: {type: 'doc', id: 'guides/automate/index'}, items: [ - 'guides/schedules', - 'guides/sensors', - 'guides/asset-sensors', - 'guides/automation', - //'guides/declarative-automation', + { + type: 'autogenerated', + dirName: 'guides/automate' + } + ], + }, + { + type: 'category', + label: 'Monitor', + collapsed: false, + link: {type: 'doc', id: 'guides/monitor/index'}, + items: [ + { + type: 'autogenerated', + dirName: 'guides/monitor' + }, ], }, { type: 'category', label: 'Test', collapsed: false, + link: {type: 'doc', id: 'guides/test/index'}, items: [ - 'guides/asset-checks', - 'guides/data-freshness-testing', - 'guides/unit-tests-assets-and-ops', + { + type: 'autogenerated', + dirName: 'guides/test' + } ], }, { type: 'category', label: 'Deploy', + link: {type: 'doc', id: 'guides/deploy/index'}, collapsed: false, items: [ { - type: 'category', - label: 'Self-hosting Dagster', - items: ['guides/kubernetes', 'guides/docker', 'guides/dagster-service'], - }, - { - type: 'category', - label: 'Dagster Plus', - items: [ - 'guides/dagster-plus', - //'guides/self-hosted-to-dagster-plus' - ], - }, - //'guides/secrets', - //'guides/code-locations', + type: 'autogenerated', + dirName: 'guides/deploy' + } ], }, - //{ - // type: 'category', - // label: 'Understand', - // items: [ - // { - // type: 'category', - // label: 'Assets', - // items: [ - // { - // type: 'autogenerated', - // dirName: 'concepts/assets', - // }, - // ], - // }, - // { - // type: 'category', - // label: 'Automation and Scheduling', - // items: [ - // { - // type: 'autogenerated', - // dirName: 'concepts/automation', - // }, - // ], - // }, - // { - // type: 'doc', - // label: 'Partitions', - // id: 'concepts/partitions', - // }, - // { - // type: 'doc', - // label: 'Resources', - // id: 'concepts/resources', - // }, - // { - // type: 'doc', - // label: 'I/O managers', - // id: 'concepts/io-managers', - // }, - // { - // type: 'category', - // label: 'Ops and jobs', - // link: { - // type: 'doc', - // id: 'concepts/ops-jobs', - // }, - // items: [ - // { - // type: 'autogenerated', - // dirName: 'concepts/ops-jobs', - // }, - // ], - // }, - // { - // type: 'category', - // label: 'Execution', - // link: { - // type: 'doc', - // id: 'concepts/execution', - // }, - // items: [ - // { - // type: 'autogenerated', - // dirName: 'concepts/execution', - // }, - // ], - // }, - // ], - //}, { type: 'category', label: 'About', @@ -307,147 +217,18 @@ const sidebars: SidebarsConfig = { }, ], dagsterPlus: [ - { - type: 'category', - label: 'Getting started', - collapsible: false, - className: 'category-non-collapsible', - items: [ - { - type: 'doc', - id: 'dagster-plus/whats-dagster-plus', - }, - { - type: 'doc', - id: 'dagster-plus/getting-started', - }, - ], - }, + 'dagster-plus/index', + 'dagster-plus/getting-started', { type: 'category', label: 'Features', collapsible: false, items: [ { - type: 'category', - label: 'Insights', - link: { - type: 'doc', - id: 'dagster-plus/insights', - }, - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/insights', - }, - ], - }, - { - type: 'category', - label: 'Branch Deployments (CI)', - link: { - type: 'doc', - id: 'dagster-plus/deployment/branch-deployments', - }, - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/deployment/branch-deployments', - }, - ], - }, - { - type: 'category', - label: 'Alerts', - link: { - type: 'doc', - id: 'dagster-plus/deployment/alerts', - }, - items: [ - { - type: 'doc', - label: 'Manage alerts in the UI', - id: 'dagster-plus/deployment/alerts/ui', - }, - { - type: 'doc', - label: 'Manage alerts with the CLI', - id: 'dagster-plus/deployment/alerts/cli', - }, - { - type: 'doc', - label: 'Email', - id: 'dagster-plus/deployment/alerts/email', - }, - { - type: 'doc', - label: 'Microsoft Teams', - id: 'dagster-plus/deployment/alerts/microsoft-teams', - }, - { - type: 'doc', - label: 'PagerDuty', - id: 'dagster-plus/deployment/alerts/pagerduty', - }, - { - type: 'doc', - label: 'Slack', - id: 'dagster-plus/deployment/alerts/slack', - }, - ], - }, - { - type: 'category', - label: 'Authentication & access control', - items: [ - { - type: 'category', - label: 'Role-based Access Control', - link: { - type: 'doc', - id: 'dagster-plus/access/rbac', - }, - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/access/rbac', - }, - ], - }, - { - type: 'category', - label: 'Single Sign-on (SSO)', - items: [ - 'dagster-plus/access/authentication/azure-ad-sso', - 'dagster-plus/access/authentication/google-workspace-sso', - 'dagster-plus/access/authentication/okta-sso', - 'dagster-plus/access/authentication/onelogin-sso', - 'dagster-plus/access/authentication/pingone-sso', - ], - }, - { - type: 'category', - label: 'SCIM provisioning', - items: [ - { - type: 'link', - label: 'Azure Active Directory', - href: 'https://learn.microsoft.com/en-us/azure/active-directory/saas-apps/dagster-cloud-provisioning-tutorial', - }, - { - type: 'doc', - label: 'Okta', - id: 'dagster-plus/access/authentication/okta-scim', - }, - ], - }, - ], - }, - { - type: 'doc', - id: 'dagster-plus/saved-views', - }, - ], + type: 'autogenerated', + dirName: 'dagster-plus/features' + } + ] }, { type: 'category', @@ -455,126 +236,26 @@ const sidebars: SidebarsConfig = { collapsible: false, items: [ { - type: 'category', - label: 'Serverless', - link: { - type: 'doc', - id: 'dagster-plus/deployment/serverless', - }, - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/deployment/serverless', - }, - ], - }, - { - type: 'category', - label: 'Hybrid', - link: { - type: 'doc', - id: 'dagster-plus/deployment/hybrid', - }, - items: [ - { - type: 'doc', - label: 'Tokens', - id: 'dagster-plus/deployment/hybrid/tokens', - }, - { - type: 'category', - label: 'Agents', - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/deployment/hybrid/agents', - }, - ], - }, - ], - }, - { - type: 'category', - label: 'CI/CD', - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/deployment/branch-deployments', - }, - ], - }, - { - type: 'category', - label: 'Code locations', - link: { - type: 'doc', - id: 'dagster-plus/deployment/code-locations', - }, - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/deployment/code-locations', - }, - ], - }, - { - type: 'category', - label: 'Environment variables', - link: { - type: 'doc', - id: 'dagster-plus/deployment/environment-variables', - }, - items: [ - { - type: 'autogenerated', - dirName: 'dagster-plus/deployment/environment-variables', - }, - ], - }, - { - type: 'doc', - label: 'Settings', - id: 'dagster-plus/settings', - }, + type: 'autogenerated', + dirName: 'dagster-plus/deployment' + } ], }, ], api: [ + 'reference/api-lifecycle', { type: 'category', - label: 'Reference', - link: {type: 'doc', id: 'reference/index'}, - collapsible: false, + label: 'Python API', collapsed: false, + link: {type: 'doc', id: 'api/index'}, items: [ { - type: 'doc', - id: 'reference/api-lifecycle', - }, - { - type: 'category', - label: 'Configuration', - collapsed: false, - items: [ - { - type: 'doc', - label: 'Dagster YAML', - id: 'reference/dagster-yaml', - }, - ], - }, - { - type: 'category', - label: 'Python API', - items: [ - { - type: 'autogenerated', - dirName: 'api', - }, - ], + type: 'autogenerated', + dirName: 'api', }, ], - }, + } ], };