diff --git a/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in new file mode 100644 index 00000000..2b12dd08 --- /dev/null +++ b/build_artifacts/v2/v2.3/v2.3.0/cpu.additional_packages_env.in @@ -0,0 +1,2 @@ +conda-forge::s3fs +conda-forge::seaborn diff --git a/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in new file mode 100644 index 00000000..2b12dd08 --- /dev/null +++ b/build_artifacts/v2/v2.3/v2.3.0/gpu.additional_packages_env.in @@ -0,0 +1,2 @@ +conda-forge::s3fs +conda-forge::seaborn diff --git a/test/test_artifacts/v2/s3fs.test.Dockerfile b/test/test_artifacts/v2/s3fs.test.Dockerfile new file mode 100644 index 00000000..8f3a0c68 --- /dev/null +++ b/test/test_artifacts/v2/s3fs.test.Dockerfile @@ -0,0 +1,10 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN git clone --recursive https://github.com/fsspec/s3fs.git + +WORKDIR "s3fs" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_s3fs_tests.sh . +RUN chmod +x run_s3fs_tests.sh +CMD ["./run_s3fs_tests.sh"] diff --git a/test/test_artifacts/v2/scripts/run_s3fs_tests.sh b/test/test_artifacts/v2/scripts/run_s3fs_tests.sh new file mode 100644 index 00000000..84571f2c --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_s3fs_tests.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# We need to checkout the version of s3fs that is installed in the mamba environment. +s3fs_version=$(micromamba list | grep s3fs | tr -s ' ' | cut -d ' ' -f 3) +# Checkout the corresponding s3fs source code +git checkout tags/$s3fs_version + +# Install requirements +# Remove portion incompatible with conda/mamba +sed -i "s/; python_version < '3.3'//" test_requirements.txt +# For <2024.10.0, there's no upperbound on moto. +# This causes test failures, so we add the upperbound. +# https://github.com/fsspec/s3fs/pull/917/files#diff-25d7875059b9486b7912948b78c0bb1c763d565fe0d12dc8287b5f0dc9982176L17 +if [[ "$s3fs_version" == "2024.10.0" ]]; then + sed -i "s/moto>=4/moto>=4,<5/" test_requirements.txt +fi +micromamba install --freeze-installed -y --file test_requirements.txt + +pytest || exit $? \ No newline at end of file diff --git a/test/test_artifacts/v2/scripts/run_seaborn_tests.sh b/test/test_artifacts/v2/scripts/run_seaborn_tests.sh new file mode 100644 index 00000000..c2c17536 --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_seaborn_tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# We need to checkout the version of seaborn that is installed in the mamba environment. +seaborn_version=$(micromamba list | grep seaborn-base | tr -s ' ' | cut -d ' ' -f 3) +# Checkout the corresponding seaborn source code +git checkout tags/v$seaborn_version + +# There's a test issue that was fixed for 0.13.2 release but isn't in the tag +# https://github.com/mwaskom/seaborn/pull/3802 +# Cherry pick works but fails to commit due to no git creds, for our test we can ignore the error. +if [[ "$seaborn_version" == "0.13.2" ]]; then + git cherry-pick 385e54676ca16d0132434bc9df6bc41ea8b2a0d4 || true +fi + +# Install test dependencies +micromamba install --freeze-installed -y pytest pytest-cov pytest-xdist pandas-stubs + +pytest || exit $? \ No newline at end of file diff --git a/test/test_artifacts/v2/seaborn.test.Dockerfile b/test/test_artifacts/v2/seaborn.test.Dockerfile new file mode 100644 index 00000000..e2a66ec5 --- /dev/null +++ b/test/test_artifacts/v2/seaborn.test.Dockerfile @@ -0,0 +1,10 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN git clone --recursive https://github.com/mwaskom/seaborn.git + +WORKDIR "seaborn" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_seaborn_tests.sh . +RUN chmod +x run_seaborn_tests.sh +CMD ["./run_seaborn_tests.sh"] diff --git a/test/test_dockerfile_based_harness.py b/test/test_dockerfile_based_harness.py index 9f416038..ebdbf032 100644 --- a/test/test_dockerfile_based_harness.py +++ b/test/test_dockerfile_based_harness.py @@ -64,6 +64,8 @@ ["jupyter-activity-monitor-extension"], ), ("docker-cli.test.Dockerfile", ["docker-cli"]), + ("s3fs.test.Dockerfile", ["s3fs"]), + ("seaborn.test.Dockerfile", ["seaborn"]), ("sagemaker-recovery-mode.test.Dockerfile", ["sagemaker-jupyterlab-extension"]), ], ) @@ -128,6 +130,8 @@ def test_dockerfiles_for_cpu( ), ("gpu-dependencies.test.Dockerfile", ["pytorch", "tensorflow"]), ("docker-cli.test.Dockerfile", ["docker-cli"]), + ("s3fs.test.Dockerfile", ["s3fs"]), + ("seaborn.test.Dockerfile", ["seaborn"]), ("sagemaker-recovery-mode.test.Dockerfile", ["sagemaker-jupyterlab-extension"]), ], )