From dd9a28dfa6f84bad1f87a4db7c1c963b0128c8fe Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 9 Dec 2024 13:16:34 -0800 Subject: [PATCH 01/10] syntax Signed-off-by: Yee Hing Tong --- .../advanced_composition/eager_workflows.py | 55 +++++++------------ mems/test.csv | 2 + 2 files changed, 21 insertions(+), 36 deletions(-) create mode 100644 mems/test.csv diff --git a/examples/advanced_composition/advanced_composition/eager_workflows.py b/examples/advanced_composition/advanced_composition/eager_workflows.py index 8a55977e5..8f16b3072 100644 --- a/examples/advanced_composition/advanced_composition/eager_workflows.py +++ b/examples/advanced_composition/advanced_composition/eager_workflows.py @@ -1,5 +1,4 @@ -from flytekit import task, workflow -from flytekit.experimental import eager +from flytekit import task, workflow, eager # Example 1 @@ -15,21 +14,21 @@ def double(x: int) -> int: @eager async def simple_eager_workflow(x: int) -> int: - out = await add_one(x=x) + out = add_one(x=x) if out < 0: return -1 - return await double(x=out) + return double(x=out) # Example 2 @eager async def another_eager_workflow(x: int) -> int: - out = await add_one(x=x) + out = add_one(x=x) # out is a Python integer out = out - 1 - return await double(x=out) + return double(x=out) # Example 3 @@ -40,14 +39,14 @@ def gt_100(x: int) -> bool: @eager async def eager_workflow_with_conditionals(x: int) -> int: - out = await add_one(x=x) + out = add_one(x=x) if out < 0: return -1 - elif await gt_100(x=out): + elif gt_100(x=out): return 100 else: - out = await double(x=out) + out = double(x=out) assert out >= -1 return out @@ -57,16 +56,20 @@ async def eager_workflow_with_conditionals(x: int) -> int: # Gather the outputs of multiple tasks or subworkflows into a list: import asyncio +@task +async def add_one_async(x: int) -> int: + return x + 1 + @eager async def eager_workflow_with_for_loop(x: int) -> int: outputs = [] for i in range(x): - outputs.append(add_one(x=i)) + outputs.append(add_one_async(x=i)) outputs = await asyncio.gather(*outputs) - return await double(x=sum(outputs)) + return double(x=sum(outputs)) # Example 5 @@ -79,27 +82,27 @@ def subworkflow(x: int) -> int: @eager async def eager_workflow_with_static_subworkflow(x: int) -> int: - out = await subworkflow(x=x) + out = subworkflow(x=x) assert out == (x + 1) * 2 return out # Example 6 -# Eager subworkflows +# Nested eager tasks @eager async def eager_subworkflow(x: int) -> int: - return await add_one(x=x) + return add_one(x=x) @eager async def nested_eager_workflow(x: int) -> int: out = await eager_subworkflow(x=x) - return await double(x=out) + return double(x=out) # Example 7 # Catching exceptions -from flytekit.experimental import EagerException +from flytekit.exceptions.eager import EagerException @task @@ -123,23 +126,3 @@ async def eager_workflow_with_exception(x: int) -> int: if __name__ == "__main__": result = asyncio.run(simple_eager_workflow(x=5)) print(f"Result: {result}") # "Result: 12" - - -# Sandbox Flyte cluster execution -# See docs for full steps -from flytekit.configuration import Config -from flytekit.remote import FlyteRemote - - -@eager( - remote=FlyteRemote( - config=Config.for_sandbox(), - default_project="flytesnacks", - default_domain="development", - ) -) -async def eager_workflow_sandbox(x: int) -> int: - out = await add_one(x=x) - if out < 0: - return -1 - return await double(x=out) diff --git a/mems/test.csv b/mems/test.csv new file mode 100644 index 000000000..8cd5d0071 --- /dev/null +++ b/mems/test.csv @@ -0,0 +1,2 @@ +a,b +3,4 From 810976f031662ae5d1f349d3062685e6f559abc6 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 9 Dec 2024 14:19:40 -0800 Subject: [PATCH 02/10] oops Signed-off-by: Yee Hing Tong --- .../advanced_composition/pub_sdk_sha.yaml | 19 +++++++++++++++++++ mems/test.csv | 2 -- 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 examples/advanced_composition/pub_sdk_sha.yaml delete mode 100644 mems/test.csv diff --git a/examples/advanced_composition/pub_sdk_sha.yaml b/examples/advanced_composition/pub_sdk_sha.yaml new file mode 100644 index 000000000..6d96cbd4e --- /dev/null +++ b/examples/advanced_composition/pub_sdk_sha.yaml @@ -0,0 +1,19 @@ +python_version: 3.11 +name: yt_public +registry: ghcr.io/wild-endeavor +platform: linux/arm64 +builder: default +apt_packages: + - "git" + - "curl" +#packages: +# - "union==0.1.82" +packages: + - pandas + - pyarrow + - plotly + - pillow + - py-spy + - flytekitplugins-deck-standard +# - "flytekit==v1.14.0b1" + - "git+https://github.com/flyteorg/flytekit.git@3071f56a353283dd56f9f5e58635c76cd1a5bfab" diff --git a/mems/test.csv b/mems/test.csv deleted file mode 100644 index 8cd5d0071..000000000 --- a/mems/test.csv +++ /dev/null @@ -1,2 +0,0 @@ -a,b -3,4 From be0fc36c6b722a2aebfc012871a7a2c3d90a48c9 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 9 Dec 2024 14:20:11 -0800 Subject: [PATCH 03/10] remove yaml Signed-off-by: Yee Hing Tong --- .../advanced_composition/pub_sdk_sha.yaml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 examples/advanced_composition/pub_sdk_sha.yaml diff --git a/examples/advanced_composition/pub_sdk_sha.yaml b/examples/advanced_composition/pub_sdk_sha.yaml deleted file mode 100644 index 6d96cbd4e..000000000 --- a/examples/advanced_composition/pub_sdk_sha.yaml +++ /dev/null @@ -1,19 +0,0 @@ -python_version: 3.11 -name: yt_public -registry: ghcr.io/wild-endeavor -platform: linux/arm64 -builder: default -apt_packages: - - "git" - - "curl" -#packages: -# - "union==0.1.82" -packages: - - pandas - - pyarrow - - plotly - - pillow - - py-spy - - flytekitplugins-deck-standard -# - "flytekit==v1.14.0b1" - - "git+https://github.com/flyteorg/flytekit.git@3071f56a353283dd56f9f5e58635c76cd1a5bfab" From aba15792f9ed0256a3dccc08ae829b9112ed0823 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 9 Dec 2024 14:22:44 -0800 Subject: [PATCH 04/10] fmt Signed-off-by: Yee Hing Tong --- .../advanced_composition/eager_workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/advanced_composition/advanced_composition/eager_workflows.py b/examples/advanced_composition/advanced_composition/eager_workflows.py index 8f16b3072..17f8d1735 100644 --- a/examples/advanced_composition/advanced_composition/eager_workflows.py +++ b/examples/advanced_composition/advanced_composition/eager_workflows.py @@ -1,4 +1,4 @@ -from flytekit import task, workflow, eager +from flytekit import eager, task, workflow # Example 1 @@ -56,6 +56,7 @@ async def eager_workflow_with_conditionals(x: int) -> int: # Gather the outputs of multiple tasks or subworkflows into a list: import asyncio + @task async def add_one_async(x: int) -> int: return x + 1 From 123271efbf2d500f7d3761d170d3fb52db409108 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 9 Dec 2024 15:11:45 -0800 Subject: [PATCH 05/10] remove old batch size example Signed-off-by: Yee Hing Tong --- .../data_types_and_io/pickle_type.py | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/examples/data_types_and_io/data_types_and_io/pickle_type.py b/examples/data_types_and_io/data_types_and_io/pickle_type.py index f0122aedf..8be1e5c5f 100644 --- a/examples/data_types_and_io/data_types_and_io/pickle_type.py +++ b/examples/data_types_and_io/data_types_and_io/pickle_type.py @@ -26,39 +26,6 @@ def superhero_wf(name: str = "Thor", power: str = "Flight") -> str: return greet_superhero(superhero=superhero) -# Batch size -# By default, if the list subtype is unrecognized, a single pickle file is generated. -# To optimize serialization and deserialization performance for scenarios involving a large number of items -# or significant list elements, you can specify a batch size. -# This feature allows for the processing of each batch as a separate pickle file. -# The following example demonstrates how to set the batch size. -from typing import Iterator - -from flytekit.types.pickle.pickle import BatchSize -from typing_extensions import Annotated - - -@task -def welcome_superheroes(names: list[str], powers: list[str]) -> Annotated[list[Superhero], BatchSize(3)]: - return [Superhero(name, power) for name, power in zip(names, powers)] - - -@task -def greet_superheroes(superheroes: list[Superhero]) -> Iterator[str]: - for superhero in superheroes: - yield f"👋 Hello {superhero.name}! Your superpower is {superhero.power}." - - -@workflow -def superheroes_wf( - names: list[str] = ["Thor", "Spiderman", "Hulk"], - powers: list[str] = ["Flight", "Surface clinger", "Shapeshifting"], -) -> Iterator[str]: - superheroes = welcome_superheroes(names=names, powers=powers) - return greet_superheroes(superheroes=superheroes) - - # Run the workflow locally if __name__ == "__main__": print(f"Superhero wf: {superhero_wf()}") - print(f"Superhero(es) wf: {superheroes_wf()}") From 6c764c2cffe25003a64740f23102275f51d19e9d Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 9 Dec 2024 22:43:08 -0800 Subject: [PATCH 06/10] upgrade lock files Signed-off-by: Yee Hing Tong --- dev-requirements.txt | 24 +++++++++++++----------- docs-requirements.txt | 19 ++++++------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 6716a9987..5337124e1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile dev-requirements.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile dev-requirements.in -o dev-requirements.txt adlfs==2024.7.0 # via flytekit aiobotocore==2.15.1 @@ -20,6 +16,8 @@ aioitertools==0.12.0 # via aiobotocore aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic attrs==24.2.0 # via # aiohttp @@ -72,7 +70,7 @@ cryptography==43.0.1 # azure-storage-blob # msal # pyjwt -dask[array,dataframe]==2024.9.1 +dask==2024.9.1 # via # -r dev-requirements.in # dask-expr @@ -272,16 +270,18 @@ pyasn1-modules==0.4.1 # via google-auth pycparser==2.22 # via cffi +pydantic==2.10.3 + # via -r dev-requirements.in +pydantic-core==2.27.1 + # via pydantic pyflakes==3.2.0 # via autoflake pygments==2.18.0 # via # flytekit # rich -pyjwt[crypto]==2.9.0 - # via - # msal - # pyjwt +pyjwt==2.9.0 + # via msal pytest==8.3.3 # via -r dev-requirements.in python-dateutil==2.9.0.post0 @@ -346,6 +346,8 @@ typing-extensions==4.12.2 # flytekit # mashumaro # mypy + # pydantic + # pydantic-core # rich-click # typing-inspect typing-inspect==0.9.0 diff --git a/docs-requirements.txt b/docs-requirements.txt index 8df2d7a24..ecd9f1863 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile docs-requirements.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile docs-requirements.in -o docs-requirements.txt accessible-pygments==0.0.5 # via pydata-sphinx-theme adlfs==2024.7.0 @@ -403,10 +399,8 @@ pygments==2.18.0 # sphinx # sphinx-prompt # sphinx-tabs -pyjwt[crypto]==2.9.0 - # via - # msal - # pyjwt +pyjwt==2.9.0 + # via msal python-dateutil==2.9.0.post0 # via # botocore @@ -475,6 +469,8 @@ scikit-learn==1.5.2 # via -r docs-requirements.in scipy==1.14.1 # via scikit-learn +setuptools==75.6.0 + # via torch six==1.16.0 # via # asttokens @@ -609,6 +605,3 @@ yarl==1.13.1 # via aiohttp zipp==3.20.2 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 7e036a764fb86f8fcfac0ec761c1528d405f2d9a Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 9 Dec 2024 22:46:18 -0800 Subject: [PATCH 07/10] use flytekit master Signed-off-by: Yee Hing Tong --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f88c21953..6f2dd40a2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -90,7 +90,7 @@ jobs: uv venv $GITHUB_WORKSPACE/.venv source $GITHUB_WORKSPACE/.venv/bin/activate if [ -f requirements.in ]; then uv pip install -r requirements.in; fi - uv pip install "flytekit>=1.12.2" "numpy<2.0.0" + uv pip install "git+https://github.com/flyteorg/flytekit.git@276c4646b209dcaf56768796dffbf772da95b784#egg=flytekit" "numpy<2.0.0" pip freeze - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 1a84a2c2e84dcb112c4ea73db29f06c241f9279b Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Tue, 10 Dec 2024 14:40:28 -0800 Subject: [PATCH 08/10] use alpha flytekit Signed-off-by: Yee Hing Tong --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6f2dd40a2..982296619 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -90,7 +90,7 @@ jobs: uv venv $GITHUB_WORKSPACE/.venv source $GITHUB_WORKSPACE/.venv/bin/activate if [ -f requirements.in ]; then uv pip install -r requirements.in; fi - uv pip install "git+https://github.com/flyteorg/flytekit.git@276c4646b209dcaf56768796dffbf772da95b784#egg=flytekit" "numpy<2.0.0" + uv pip install "flytekit>=1.15.0a0" "numpy<2.0.0" pip freeze - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 6d971ae7f1e6af9400a9d379eabe36dcb811b378 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 11 Dec 2024 12:49:33 -0500 Subject: [PATCH 09/10] Use flytekit 1.15.0a1 Signed-off-by: Eduardo Apolinario --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 982296619..0afd2ad02 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -90,7 +90,7 @@ jobs: uv venv $GITHUB_WORKSPACE/.venv source $GITHUB_WORKSPACE/.venv/bin/activate if [ -f requirements.in ]; then uv pip install -r requirements.in; fi - uv pip install "flytekit>=1.15.0a0" "numpy<2.0.0" + uv pip install "flytekit>=1.15.0a1" "numpy<2.0.0" pip freeze - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 37935d003e70cfb02dc4aab5c8b1ce7575cf1220 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Wed, 11 Dec 2024 10:34:38 -0800 Subject: [PATCH 10/10] bump version Signed-off-by: Yee Hing Tong --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 982296619..0afd2ad02 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -90,7 +90,7 @@ jobs: uv venv $GITHUB_WORKSPACE/.venv source $GITHUB_WORKSPACE/.venv/bin/activate if [ -f requirements.in ]; then uv pip install -r requirements.in; fi - uv pip install "flytekit>=1.15.0a0" "numpy<2.0.0" + uv pip install "flytekit>=1.15.0a1" "numpy<2.0.0" pip freeze - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3