From 81253b6893a812ab04225431b6bbbbb72596f897 Mon Sep 17 00:00:00 2001 From: Clayton Parnell Date: Tue, 7 Jan 2025 19:46:37 +0000 Subject: [PATCH 1/5] add additional packages --- .../v2/v2.3/v2.3.0/cpu.additional_packages_env.in | 5 +++++ .../v2/v2.3/v2.3.0/gpu.additional_packages_env.in | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in create mode 100644 build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in diff --git a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in new file mode 100644 index 00000000..504b6ec7 --- /dev/null +++ b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in @@ -0,0 +1,5 @@ +conda-forge::s3fs +conda-forge::aioboto3 +conda-forge::llamaindex-py-client +conda-forge::seaborn + diff --git a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in new file mode 100644 index 00000000..504b6ec7 --- /dev/null +++ b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in @@ -0,0 +1,5 @@ +conda-forge::s3fs +conda-forge::aioboto3 +conda-forge::llamaindex-py-client +conda-forge::seaborn + From e84b1b28ef4965c80fa9df6dcc6e0b738eebb5e1 Mon Sep 17 00:00:00 2001 From: Clayton Parnell Date: Tue, 7 Jan 2025 21:00:46 +0000 Subject: [PATCH 2/5] remove aioboto3 --- build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in | 1 - build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in | 1 - 2 files changed, 2 deletions(-) diff --git a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in index 504b6ec7..57a7a423 100644 --- a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in @@ -1,5 +1,4 @@ conda-forge::s3fs -conda-forge::aioboto3 conda-forge::llamaindex-py-client conda-forge::seaborn diff --git a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in index 504b6ec7..57a7a423 100644 --- a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in @@ -1,5 +1,4 @@ conda-forge::s3fs -conda-forge::aioboto3 conda-forge::llamaindex-py-client conda-forge::seaborn From 6cd085bf4913dc57ff8f6b966f567f2ec5417e0e Mon Sep 17 00:00:00 2001 From: Clayton Parnell Date: Wed, 8 Jan 2025 18:57:34 +0000 Subject: [PATCH 3/5] Update llama-index pkg --- build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in | 3 +-- build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in index 57a7a423..52d95c3f 100644 --- a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in @@ -1,4 +1,3 @@ conda-forge::s3fs -conda-forge::llamaindex-py-client +conda-forge::llama-index conda-forge::seaborn - diff --git a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in index 57a7a423..52d95c3f 100644 --- a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in @@ -1,4 +1,3 @@ conda-forge::s3fs -conda-forge::llamaindex-py-client +conda-forge::llama-index conda-forge::seaborn - From a115b18fb9c738f6580a929c5a6f2f9d31b835ed Mon Sep 17 00:00:00 2001 From: Clayton Parnell Date: Fri, 10 Jan 2025 21:50:10 +0000 Subject: [PATCH 4/5] Remove llama-index; add tests --- .../v2.3.0/cpu.additional_packages_env.in | 1 - .../v2.3.0/gpu.additional_packages_env.in | 1 - test/test_artifacts/v2/s3fs.test.Dockerfile | 10 ++++++++++ .../v2/scripts/run_s3fs_tests.sh | 19 +++++++++++++++++++ .../v2/scripts/run_seaborn_tests.sh | 18 ++++++++++++++++++ .../test_artifacts/v2/seaborn.test.Dockerfile | 10 ++++++++++ test/test_dockerfile_based_harness.py | 4 ++++ 7 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 test/test_artifacts/v2/s3fs.test.Dockerfile create mode 100644 test/test_artifacts/v2/scripts/run_s3fs_tests.sh create mode 100644 test/test_artifacts/v2/scripts/run_seaborn_tests.sh create mode 100644 test/test_artifacts/v2/seaborn.test.Dockerfile diff --git a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in index 52d95c3f..2b12dd08 100644 --- a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in @@ -1,3 +1,2 @@ conda-forge::s3fs -conda-forge::llama-index conda-forge::seaborn diff --git a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in index 52d95c3f..2b12dd08 100644 --- a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in @@ -1,3 +1,2 @@ conda-forge::s3fs -conda-forge::llama-index conda-forge::seaborn diff --git a/test/test_artifacts/v2/s3fs.test.Dockerfile b/test/test_artifacts/v2/s3fs.test.Dockerfile new file mode 100644 index 00000000..8f3a0c68 --- /dev/null +++ b/test/test_artifacts/v2/s3fs.test.Dockerfile @@ -0,0 +1,10 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN git clone --recursive https://github.com/fsspec/s3fs.git + +WORKDIR "s3fs" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_s3fs_tests.sh . +RUN chmod +x run_s3fs_tests.sh +CMD ["./run_s3fs_tests.sh"] diff --git a/test/test_artifacts/v2/scripts/run_s3fs_tests.sh b/test/test_artifacts/v2/scripts/run_s3fs_tests.sh new file mode 100644 index 00000000..ad2b40fb --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_s3fs_tests.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# We need to checkout the version of s3fs that is installed in the mamba environment. +s3fs_version=$(micromamba list | grep s3fs | tr -s ' ' | cut -d ' ' -f 3) +# Checkout the corresponding s3fs source code +git checkout tags/$s3fs_version + +# Install requirements +# Remove portion incompatible with conda/mamba +sed -i "s/; python_version < '3.3'//" test_requirements.txt +# For <2024.12.0, there's no upperbound on moto. +# This causes test failures, so we add the upperbound. +# https://github.com/fsspec/s3fs/pull/917/files#diff-25d7875059b9486b7912948b78c0bb1c763d565fe0d12dc8287b5f0dc9982176L17 +if [[ "$s3fs_version" == "2024.10.0" ]]; then + sed -i "s/moto>=4/moto>=4,<5/" test_requirements.txt +fi +micromamba install --freeze-installed -y --file test_requirements.txt + +pytest || exit $? \ No newline at end of file diff --git a/test/test_artifacts/v2/scripts/run_seaborn_tests.sh b/test/test_artifacts/v2/scripts/run_seaborn_tests.sh new file mode 100644 index 00000000..c2c17536 --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_seaborn_tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# We need to checkout the version of seaborn that is installed in the mamba environment. +seaborn_version=$(micromamba list | grep seaborn-base | tr -s ' ' | cut -d ' ' -f 3) +# Checkout the corresponding seaborn source code +git checkout tags/v$seaborn_version + +# There's a test issue that was fixed for 0.13.2 release but isn't in the tag +# https://github.com/mwaskom/seaborn/pull/3802 +# Cherry pick works but fails to commit due to no git creds, for our test we can ignore the error. +if [[ "$seaborn_version" == "0.13.2" ]]; then + git cherry-pick 385e54676ca16d0132434bc9df6bc41ea8b2a0d4 || true +fi + +# Install test dependencies +micromamba install --freeze-installed -y pytest pytest-cov pytest-xdist pandas-stubs + +pytest || exit $? \ No newline at end of file diff --git a/test/test_artifacts/v2/seaborn.test.Dockerfile b/test/test_artifacts/v2/seaborn.test.Dockerfile new file mode 100644 index 00000000..e2a66ec5 --- /dev/null +++ b/test/test_artifacts/v2/seaborn.test.Dockerfile @@ -0,0 +1,10 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN git clone --recursive https://github.com/mwaskom/seaborn.git + +WORKDIR "seaborn" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_seaborn_tests.sh . +RUN chmod +x run_seaborn_tests.sh +CMD ["./run_seaborn_tests.sh"] diff --git a/test/test_dockerfile_based_harness.py b/test/test_dockerfile_based_harness.py index e23b40a7..72a43adf 100644 --- a/test/test_dockerfile_based_harness.py +++ b/test/test_dockerfile_based_harness.py @@ -48,6 +48,8 @@ ("mlflow.test.Dockerfile", ["mlflow"]), ("jupyter-activity-monitor-extension.test.Dockerfile", ["jupyter-activity-monitor-extension"]), ("docker-cli.test.Dockerfile", ["docker-cli"]), + ("s3fs.test.Dockerfile", ["s3fs"]), + ("seaborn.test.Dockerfile", ["seaborn"]), ], ) def test_dockerfiles_for_cpu( @@ -93,6 +95,8 @@ def test_dockerfiles_for_cpu( ("jupyter-activity-monitor-extension.test.Dockerfile", ["jupyter-activity-monitor-extension"]), ("gpu-dependencies.test.Dockerfile", ["pytorch", "tensorflow"]), ("docker-cli.test.Dockerfile", ["docker-cli"]), + ("s3fs.test.Dockerfile", ["s3fs"]), + ("seaborn.test.Dockerfile", ["seaborn"]), ], ) def test_dockerfiles_for_gpu( From 3a779af1a8c8724fc0b8d484d1eac5fde4d90959 Mon Sep 17 00:00:00 2001 From: Clayton Parnell Date: Tue, 14 Jan 2025 17:44:06 +0000 Subject: [PATCH 5/5] fix typo --- test/test_artifacts/v2/scripts/run_s3fs_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_artifacts/v2/scripts/run_s3fs_tests.sh b/test/test_artifacts/v2/scripts/run_s3fs_tests.sh index ad2b40fb..84571f2c 100644 --- a/test/test_artifacts/v2/scripts/run_s3fs_tests.sh +++ b/test/test_artifacts/v2/scripts/run_s3fs_tests.sh @@ -8,7 +8,7 @@ git checkout tags/$s3fs_version # Install requirements # Remove portion incompatible with conda/mamba sed -i "s/; python_version < '3.3'//" test_requirements.txt -# For <2024.12.0, there's no upperbound on moto. +# For <2024.10.0, there's no upperbound on moto. # This causes test failures, so we add the upperbound. # https://github.com/fsspec/s3fs/pull/917/files#diff-25d7875059b9486b7912948b78c0bb1c763d565fe0d12dc8287b5f0dc9982176L17 if [[ "$s3fs_version" == "2024.10.0" ]]; then