Merge branch 'develop' into feature/PRD-543-remove-deprecated-features

.github/workflows/update-templates-to-examples.yml

@@ -118,7 +118,7 @@ jobs:
           python-version: ${{ inputs.python-version }}
           stack-name: local
           ref-zenml: ${{ github.ref }}
-          ref-template: 2024.08.29  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
+          ref-template: 2024.09.23  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
       - name: Clean-up
         run: |
           rm -rf ./local_checkout
@@ -189,7 +189,7 @@ jobs:
           python-version: ${{ inputs.python-version }}
           stack-name: local
           ref-zenml: ${{ github.ref }}
-          ref-template: 2024.08.28  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
+          ref-template: 2024.09.23  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
       - name: Clean-up
         run: |
           rm -rf ./local_checkout

.hyperlint/config.yaml

@@ -0,0 +1 @@
+content_dir: docs/

README.md

@@ -327,7 +327,7 @@ the Apache License Version 2.0.
     <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
     <br />
     <br />
-    🎉 Version 0.66.0 is out. Check out the release notes
+    🎉 Version 0.67.0 is out. Check out the release notes
     <a href="https://github.com/zenml-io/zenml/releases">here</a>.
     <br />
     🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.

RELEASE_NOTES.md

@@ -1,5 +1,103 @@
 <!-- markdown-link-check-disable -->
 
+# 0.67.0
+
+## Highlights
+
+- **Improved Sagemaker Orchestrator:** Now supports warm pools for AWS Sagemaker, enhancing performance and reducing startup times for TrainingJobs.
+- **New DAG Visualizer:** Shipped major enhancements to the DAG Visualizer for Pipeline Runs:
+  - Preview of the actual DAG before pipeline completion
+  - Visual adjustments for improved clarity
+  - Real-time updates during pipeline execution
+- **Environment Variable References in Configurations:** Introduced the ability to reference environment variables in both code and configuration files using the syntax ${ENV_VARIABLE_NAME}, increasing flexibility in setups.
+- **Enhanced UX for Major Cloud Providers:** Displaying direct pipeline/log URL when working with major cloud platforms.
+- **Skypilot with Kubernetes Support:** Added compatibility for running Skypilot orchestrator on Kubernetes clusters.
+- **Updated Deepchecks Integration:** The Deepchecks integration has been refreshed with the latest features and improvements.
+
+
+## Features and Improvements
+
+- **AWS Integration:**
+  - Added permissions to workflow to enable assuming AWS role.
+  - Fixed expired credentials error when using the docker service connector.
+- **Error Handling:** Improved error messages for stack components of uninstalled integrations.
+- **API Key Management:** Added an option to write API keys to a file instead of using the CLI.
+
+## Pipeline Execution:
+
+- Implemented fixes for executing steps as single step pipelines.
+- Added filter option for templatable runs.
+- Added additional filtering options for pipeline runs.
+- MLflow Integration: Linked registered models in MLflow with the corresponding MLflow run.
+- Analytics: Added missing analytics event to improve user insights.
+
+## Documentation Updates
+
+- Updated documentation for various integrations including:
+  - Lightning AI orchestrator
+  - Kubeflow
+  - Comet experiment tracker
+  - Neptune
+  - Hugging Face deployer
+  - Weights & Biases (wandb)
+- Added documentation for run templates.
+- Fixed incorrect method name in Pigeon docs.
+- Various small documentation fixes and improvements.
+
+## Bug Fixes
+
+- Fixed YAML formatting issues.
+- Resolved RBAC issues for subpages in response models.
+- Fixed step output annotation in Discord test.
+- Addressed MLFlow integration requirements duplication.
+- Fixed Lightning orchestrator functionality.
+
+## What's Changed
+* Error message for stack components of uninstalled integrations by @bcdurak in https://github.com/zenml-io/zenml/pull/2996
+* Enable migration testing for version 0.66.0 by @schustmi in https://github.com/zenml-io/zenml/pull/2998
+* Add permissions to workflow to enable assuming AWS role by @schustmi in https://github.com/zenml-io/zenml/pull/2999
+* Add option to write api key to file instead of CLI by @schustmi in https://github.com/zenml-io/zenml/pull/3001
+* Fix yaml formatting by @schustmi in https://github.com/zenml-io/zenml/pull/3004
+* Update ZenML Pro links for consistency.- Update ZenML Pro links for c… by @htahir1 in https://github.com/zenml-io/zenml/pull/3007
+* Fix incorrect method name in Pigeon docs by @strickvl in https://github.com/zenml-io/zenml/pull/3008
+* Fixes for executing steps as single step pipelines by @schustmi in https://github.com/zenml-io/zenml/pull/3006
+* Add filter option for templatable runs by @schustmi in https://github.com/zenml-io/zenml/pull/3000
+* Add missing analytics event by @schustmi in https://github.com/zenml-io/zenml/pull/3009
+* Fix expired credentials error when using the docker service connector by @schustmi in https://github.com/zenml-io/zenml/pull/3002
+* Fix Lightning docs by @strickvl in https://github.com/zenml-io/zenml/pull/3013
+* Remove image builder warning by @htahir1 in https://github.com/zenml-io/zenml/pull/3014
+* Fixed kubeflow docs by @AlexejPenner in https://github.com/zenml-io/zenml/pull/3018
+* Update Comet experiment tracker docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3019
+* Small docs fixes by @strickvl in https://github.com/zenml-io/zenml/pull/3022
+* Feature/cleanup unused file by @AlexejPenner in https://github.com/zenml-io/zenml/pull/3023
+* MLFlow integration requirements duplicate fix by @bcdurak in https://github.com/zenml-io/zenml/pull/3011
+* Fix Neptune docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3026
+* Fix huggingface deployer docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3024
+* Fix step output annotation in Discord test by @wjayesh in https://github.com/zenml-io/zenml/pull/3029
+* Fix RBAC for subpages in response models by @schustmi in https://github.com/zenml-io/zenml/pull/3031
+* Allow env variable placeholders in configurations by @schustmi in https://github.com/zenml-io/zenml/pull/3003
+* Leverage warm pools for AWS Sagemaker by @avishniakov in https://github.com/zenml-io/zenml/pull/3027
+* Updated wandb docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3030
+* Add hyperlint by @htahir1 in https://github.com/zenml-io/zenml/pull/3035
+* Bump NLP template by @avishniakov in https://github.com/zenml-io/zenml/pull/3036
+* Add additional filtering options by @schustmi in https://github.com/zenml-io/zenml/pull/2951
+* Bump starter template version by @schustmi in https://github.com/zenml-io/zenml/pull/3038
+* Docs for run templates by @bcdurak in https://github.com/zenml-io/zenml/pull/3028
+* Update Lightning AI orchestrator documentation by @strickvl in https://github.com/zenml-io/zenml/pull/3016
+* Add default value for PipelineRun.is_templatable by @schustmi in https://github.com/zenml-io/zenml/pull/3040
+* Use a generic OAuth2 client credentials flow to login to the Cloud API by @stefannica in https://github.com/zenml-io/zenml/pull/3041
+* fix lightning orchestrator by @safoinme in https://github.com/zenml-io/zenml/pull/3010
+* Linking registered models in MLflow with the corresponding MLflow run by @aiakide in https://github.com/zenml-io/zenml/pull/3020
+* Bugfixing mlflow registry linting issue by @bcdurak in https://github.com/zenml-io/zenml/pull/3043
+* Enhancing the orchestrator UX for major cloud providers by @bcdurak in https://github.com/zenml-io/zenml/pull/3005
+* Skypilot with Kubernetes by @safoinme in https://github.com/zenml-io/zenml/pull/3033
+* Update deepchecks integration by @wjayesh in https://github.com/zenml-io/zenml/pull/2987
+
+## New Contributors
+* @aiakide made their first contribution in https://github.com/zenml-io/zenml/pull/3020
+
+**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.66.0...0.67.0
+
 # 0.66.0
 
 ## New Features and Improvements

docker/base.Dockerfile

@@ -76,7 +76,7 @@ RUN if [ "$ZENML_NIGHTLY" = "true" ]; then \
       PACKAGE_NAME="zenml"; \
     fi \
     && pip install --upgrade pip \
-    && pip install "${PACKAGE_NAME}[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure]${ZENML_VERSION:+==$ZENML_VERSION}" \
+    && pip install "${PACKAGE_NAME}[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure,azureml,sagemaker,vertex]${ZENML_VERSION:+==$ZENML_VERSION}" \
     && pip freeze > requirements.txt
 
 FROM base as client

docker/zenml-server-dev.Dockerfile

@@ -53,14 +53,14 @@ COPY src/zenml/__init__.py ./src/zenml/
 # NOTE: we need to uninstall zenml at the end to remove the incomplete
 # installation
 RUN pip install --upgrade pip \
-  && pip install .[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure] \
+  && pip install .[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure,azureml,sagemaker,vertex] \
   && pip freeze > requirements.txt
 
 # Copy the source code
 COPY src src
 
 # Run pip install again to install the source code in the virtual environment
-RUN pip install --no-deps --no-cache .[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure]
+RUN pip install --no-deps --no-cache .[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure,azureml,sagemaker,vertex]
 
 # Inherit from the base image which has the minimal set of updated system
 # software packages

docs/book/component-guide/annotators/pigeon.md

@@ -55,7 +55,7 @@ from zenml.client import Client
 
 annotator = Client().active_stack.annotator
 
-annotations = annotator.launch(
+annotations = annotator.annotate(
     data=[
         'I love this movie',
         'I was really disappointed by the book'
@@ -75,7 +75,7 @@ from IPython.display import display, Image
 
 annotator = Client().active_stack.annotator
 
-annotations = annotator.launch(
+annotations = annotator.annotate(
     data=[
         '/path/to/image1.png',
         '/path/to/image2.png'