From 76fb652add7cbe25319eebcf31c0320c63531e8a Mon Sep 17 00:00:00 2001 From: Hamza Tahir Date: Mon, 19 Aug 2024 21:57:18 +0200 Subject: [PATCH 1/2] Hydrated pipeline build --- src/zenml/cli/pipeline.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/zenml/cli/pipeline.py b/src/zenml/cli/pipeline.py index f5be255f100..e5a541cf059 100644 --- a/src/zenml/cli/pipeline.py +++ b/src/zenml/cli/pipeline.py @@ -516,7 +516,7 @@ def list_pipeline_builds(**kwargs: Any) -> None: client = Client() try: with console.status("Listing pipeline builds...\n"): - pipeline_builds = client.list_builds(**kwargs) + pipeline_builds = client.list_builds(hydrate=True, **kwargs) except KeyError as err: cli_utils.error(str(err)) else: @@ -526,7 +526,14 @@ def list_pipeline_builds(**kwargs: Any) -> None: cli_utils.print_pydantic_models( pipeline_builds, - exclude_columns=["created", "updated", "user", "workspace"], + exclude_columns=[ + "created", + "updated", + "user", + "workspace", + "images", + "stack_checksum", + ], ) From b68bc2a5c7d10a85ec644b1c6f5e1e716c5c6536 Mon Sep 17 00:00:00 2001 From: Hamza Tahir Date: Mon, 19 Aug 2024 22:01:36 +0200 Subject: [PATCH 2/2] removed docs for reuse docker build --- .../configure-python-environments/README.md | 2 - .../build-the-pipeline-without-running.md | 46 ------------------- .../reuse-docker-builds.md | 30 ------------ ...sitories-to-speed-up-docker-build-times.md | 2 +- docs/book/toc.md | 2 - 5 files changed, 1 insertion(+), 81 deletions(-) delete mode 100644 docs/book/how-to/customize-docker-builds/build-the-pipeline-without-running.md delete mode 100644 docs/book/how-to/customize-docker-builds/reuse-docker-builds.md diff --git a/docs/book/how-to/configure-python-environments/README.md b/docs/book/how-to/configure-python-environments/README.md index 5d4e247ac10..ad636672ddf 100644 --- a/docs/book/how-to/configure-python-environments/README.md +++ b/docs/book/how-to/configure-python-environments/README.md @@ -41,8 +41,6 @@ When running locally, there is no real concept of an `execution` environment as ZenML handles the Docker image configuration, creation, and pushing, starting with a [base image](https://hub.docker.com/r/zenmldocker/zenml) containing ZenML and Python, then adding pipeline dependencies. To manage the Docker image configuration, follow the steps in the [containerize your pipeline](../../how-to/customize-docker-builds/README.md) guide, including specifying additional pip dependencies, using a custom parent image, and customizing the build process. -The execution environments do not need to be built each time a pipeline is run - you can [reuse builds from previous runs to save time](../../how-to/customize-docker-builds/reuse-docker-builds.md). - ## Image Builder Environment By default, execution environments are created locally in the [client environment](#client-environment-or-the-runner-environment) using the local Docker client. However, this requires Docker installation and permissions. ZenML offers [image builders](../../component-guide/image-builders/image-builders.md), a special [stack component](../../component-guide/README.md), allowing users to build and push Docker images in a different specialized _image builder environment_. diff --git a/docs/book/how-to/customize-docker-builds/build-the-pipeline-without-running.md b/docs/book/how-to/customize-docker-builds/build-the-pipeline-without-running.md deleted file mode 100644 index a2d30ae1089..00000000000 --- a/docs/book/how-to/customize-docker-builds/build-the-pipeline-without-running.md +++ /dev/null @@ -1,46 +0,0 @@ -# Build the pipeline without running - -Pipeline builds are usually done implicitly when you run a pipeline on a docker-based orchestrator. But you are also able to just build a pipeline without running this. To do this you simply do this: - -```python -from zenml import pipeline - -@pipeline -def my_pipeline(...): - ... - -my_pipeline.build() -``` - -You can see all pipeline builds with the command: - -This will register the build output in the ZenML database and allow you to use the built images when running a pipeline later. - -```bash -zenml pipeline builds list -``` - -To [reuse a registered build](./reuse-docker-builds.md) when running a pipeline, pass it as an argument in Python - -```python -my_pipeline = my_pipeline.with_options(build=) -``` - -or when running a pipeline from the CLI - -```bash -zenml pipeline run --build= -``` - -{% hint style="info" %} -The source path of your pipeline will be `run.my_pipeline`. In a generalized -way, this will be `.`. If the python file -defining the pipeline is not in your current directory, the module path consists -of the full path to the file, separated by dots, e.g. -`some_directory.some_file.my_pipeline`. -{% endhint %} - - -
ZenML Scarf
- - diff --git a/docs/book/how-to/customize-docker-builds/reuse-docker-builds.md b/docs/book/how-to/customize-docker-builds/reuse-docker-builds.md deleted file mode 100644 index 32796500b13..00000000000 --- a/docs/book/how-to/customize-docker-builds/reuse-docker-builds.md +++ /dev/null @@ -1,30 +0,0 @@ -# Reuse Docker builds - -## Avoid building Docker images each time a pipeline runs - -When using containerized components in your stack, ZenML needs to [build Docker images to remotely execute your code](../configure-python-environments/README.md#execution-environments). Building Docker images without [connecting a git repository](../../user-guide/production-guide/connect-code-repository.md) **includes your step code in the built Docker image**. This, however, means that new Docker images will be built and pushed whenever you make changes to any of your source files. - -One way of skipping Docker builds each time is to pass in the ID of a `build` as you run the pipeline: - -```python -my_pipeline = my_pipeline.with_options(build=) -``` - -or when running a pipeline from the CLI: - -```shell -zenml pipeline run --build= -``` - -{% hint style="info" %} -The source path of your pipeline will be `run.my_pipeline`. In a generalized -way, this will be `.`. If the python file -defining the pipeline is not in your current directory, the module path consists -of the full path to the file, separated by dots, e.g. -`some_directory.some_file.my_pipeline`. -{% endhint %} - -Please note, that this means specifying a custom build when running a pipeline will **not run the code on your client machine** but will use the code **included in the Docker images of the build**. As a consequence, even if you make local code changes, reusing a build will _always_ execute the code bundled in the Docker image, rather than the local code. Therefore, if you would like to reuse a Docker build AND make sure your local code changes are also downloaded into the image, you need to [connect a git repository](use-code-repositories-to-speed-up-docker-build-times.md). - - -
ZenML Scarf
diff --git a/docs/book/how-to/customize-docker-builds/use-code-repositories-to-speed-up-docker-build-times.md b/docs/book/how-to/customize-docker-builds/use-code-repositories-to-speed-up-docker-build-times.md index 438289ce639..f1c53381dc7 100644 --- a/docs/book/how-to/customize-docker-builds/use-code-repositories-to-speed-up-docker-build-times.md +++ b/docs/book/how-to/customize-docker-builds/use-code-repositories-to-speed-up-docker-build-times.md @@ -1,6 +1,6 @@ # Use code repositories to speed up Docker build times -While [reusing Docker builds](reuse-docker-builds.md) is useful, it can be limited. This is because specifying a custom build when running a pipeline will **not run the code on your client machine** but will use the code **included in the Docker images of the build**. As a consequence, even if you make local code changes, reusing a build will _always_ execute the code bundled in the Docker image, rather than the local code. Therefore, if you would like to reuse a Docker build AND make sure your local code changes are also downloaded into the image, you need to disconnect your code from the build. +While reusing Docker builds is useful, it can be limited. This is because specifying a custom build when running a pipeline will **not run the code on your client machine** but will use the code **included in the Docker images of the build**. As a consequence, even if you make local code changes, reusing a build will _always_ execute the code bundled in the Docker image, rather than the local code. Therefore, if you would like to reuse a Docker build AND make sure your local code changes are also downloaded into the image, you need to disconnect your code from the build. You can do so by connecting a git repository. Registering a code repository lets you avoid building images each time you run a pipeline **and** quickly iterate on your code. When running a pipeline that is part of a local code repository checkout, ZenML can instead build the Docker images without including any of your source files, and download the files inside the container before running your code. This greatly speeds up the building process and also allows you to reuse images that one of your colleagues might have built for the same stack. diff --git a/docs/book/toc.md b/docs/book/toc.md index e34f6e7ba24..1d61cc818d7 100644 --- a/docs/book/toc.md +++ b/docs/book/toc.md @@ -106,9 +106,7 @@ * [Specify pip dependencies and apt packages](how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md) * [Use your own Dockerfiles](how-to/customize-docker-builds/use-your-own-docker-files.md) * [Which files are built into the image](how-to/customize-docker-builds/which-files-are-built-into-the-image.md) - * [Reuse Docker builds to speed up Docker build times](how-to/customize-docker-builds/reuse-docker-builds.md) * [Use code repositories to automate Docker build reuse](how-to/customize-docker-builds/use-code-repositories-to-speed-up-docker-build-times.md) - * [Build the pipeline without running](how-to/customize-docker-builds/build-the-pipeline-without-running.md) * [Define where an image is built](how-to/customize-docker-builds/define-where-an-image-is-built.md) * [⚒️ Manage stacks & components](how-to/stack-deployment/README.md) * [Deploy a cloud stack with ZenML](how-to/stack-deployment/deploy-a-cloud-stack.md)