From 5922998f8836b1d035b392749e04b0f80380a0ab Mon Sep 17 00:00:00 2001 From: Piyush Jain Date: Mon, 2 Oct 2023 19:53:00 +0000 Subject: [PATCH] Added jupyter_ai, v1 tests --- .../v1/autogluon.test.Dockerfile | 13 +++++++++ test/test_artifacts/v1/boto3.test.Dockerfile | 16 ++++++++++ .../v1/jupyter-ai.test.Dockerfile | 6 ++++ test/test_artifacts/v1/keras.test.Dockerfile | 25 ++++++++++++++++ .../v1/matplotlib.test.Dockerfile | 19 ++++++++++++ test/test_artifacts/v1/numpy.test.Dockerfile | 13 +++++++++ test/test_artifacts/v1/pandas.test.Dockerfile | 8 +++++ .../v1/pytorch.examples.Dockerfile | 20 +++++++++++++ test/test_artifacts/v1/run_pandas_tests.py | 20 +++++++++++++ test/test_artifacts/v1/scipy.test.Dockerfile | 12 ++++++++ .../v1/scripts/run_autogluon_tests.sh | 17 +++++++++++ .../v1/scripts/run_boto3_tests.sh | 10 +++++++ .../v1/scripts/run_keras_tests.sh | 11 +++++++ .../v1/scripts/run_matplotlib_tests.sh | 7 +++++ .../v1/scripts/run_pysdk_tests.sh | 20 +++++++++++++ .../v1/sm-python-sdk.test.Dockerfile | 13 +++++++++ .../v1/tensorflow.examples.Dockerfile | 16 ++++++++++ .../run_tensorflow_example_notebooks.sh | 29 +++++++++++++++++++ test/test_dockerfile_based_harness.py | 6 ++-- 19 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 test/test_artifacts/v1/autogluon.test.Dockerfile create mode 100644 test/test_artifacts/v1/boto3.test.Dockerfile create mode 100644 test/test_artifacts/v1/jupyter-ai.test.Dockerfile create mode 100644 test/test_artifacts/v1/keras.test.Dockerfile create mode 100644 test/test_artifacts/v1/matplotlib.test.Dockerfile create mode 100644 test/test_artifacts/v1/numpy.test.Dockerfile create mode 100644 test/test_artifacts/v1/pandas.test.Dockerfile create mode 100644 test/test_artifacts/v1/pytorch.examples.Dockerfile create mode 100644 test/test_artifacts/v1/run_pandas_tests.py create mode 100644 test/test_artifacts/v1/scipy.test.Dockerfile create mode 100644 test/test_artifacts/v1/scripts/run_autogluon_tests.sh create mode 100644 test/test_artifacts/v1/scripts/run_boto3_tests.sh create mode 100644 test/test_artifacts/v1/scripts/run_keras_tests.sh create mode 100644 test/test_artifacts/v1/scripts/run_matplotlib_tests.sh create mode 100644 test/test_artifacts/v1/scripts/run_pysdk_tests.sh create mode 100644 test/test_artifacts/v1/sm-python-sdk.test.Dockerfile create mode 100644 test/test_artifacts/v1/tensorflow.examples.Dockerfile create mode 100644 test/test_artifacts/v1/tensorflow/run_tensorflow_example_notebooks.sh diff --git a/test/test_artifacts/v1/autogluon.test.Dockerfile b/test/test_artifacts/v1/autogluon.test.Dockerfile new file mode 100644 index 00000000..34867534 --- /dev/null +++ b/test/test_artifacts/v1/autogluon.test.Dockerfile @@ -0,0 +1,13 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE as base + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +RUN micromamba install --freeze-installed -y conda-forge::pytest conda-forge::jupyter + +RUN git clone --recursive https://github.com/autogluon/autogluon.git + +WORKDIR "autogluon" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_autogluon_tests.sh . +RUN chmod +x run_autogluon_tests.sh +CMD ["./run_autogluon_tests.sh"] diff --git a/test/test_artifacts/v1/boto3.test.Dockerfile b/test/test_artifacts/v1/boto3.test.Dockerfile new file mode 100644 index 00000000..c0c88104 --- /dev/null +++ b/test/test_artifacts/v1/boto3.test.Dockerfile @@ -0,0 +1,16 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN sudo apt-get update && sudo apt-get install -y git && \ + git clone --recursive https://github.com/boto/boto3.git && \ + : + +# For Running boto3 tests, we need pytest +RUN micromamba install -y --freeze-installed conda-forge::pytest + + +WORKDIR "boto3" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_boto3_tests.sh . +RUN chmod +x run_boto3_tests.sh +CMD ["./run_boto3_tests.sh"] diff --git a/test/test_artifacts/v1/jupyter-ai.test.Dockerfile b/test/test_artifacts/v1/jupyter-ai.test.Dockerfile new file mode 100644 index 00000000..e231adf7 --- /dev/null +++ b/test/test_artifacts/v1/jupyter-ai.test.Dockerfile @@ -0,0 +1,6 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +CMD ["python", "-c", "import jupyter_ai"] diff --git a/test/test_artifacts/v1/keras.test.Dockerfile b/test/test_artifacts/v1/keras.test.Dockerfile new file mode 100644 index 00000000..0d03504f --- /dev/null +++ b/test/test_artifacts/v1/keras.test.Dockerfile @@ -0,0 +1,25 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +RUN sudo apt-get update && sudo apt-get install -y git graphviz && \ + git clone --recursive https://github.com/keras-team/keras-io.git && \ + : + +# Some of the keras guides requires pydot and graphviz to be installed +RUN micromamba install -y --freeze-installed conda-forge::pydot nvidia::cuda-nvcc +ENV XLA_FLAGS=--xla_gpu_cuda_data_dir=/opt/conda + +WORKDIR "keras-io/guides" + +# Checkout a specific commit known to be compatible with the runtime's current version of TensorFlow. +# keras-io made backwards incompatible changes that broke these tests. Pinning at this commit for now +# at least until the runtime's TensorFlow dependency is upgraded to the next minor version +RUN git checkout 861b59747b43ce326bb0a12384a07d6632249901 + +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_keras_tests.sh . +RUN chmod +x run_keras_tests.sh +# Run tests in run_keras_tests.sh +CMD ["./run_keras_tests.sh"] + diff --git a/test/test_artifacts/v1/matplotlib.test.Dockerfile b/test/test_artifacts/v1/matplotlib.test.Dockerfile new file mode 100644 index 00000000..95b63731 --- /dev/null +++ b/test/test_artifacts/v1/matplotlib.test.Dockerfile @@ -0,0 +1,19 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +RUN sudo apt-get update && sudo apt-get install -y git && \ + git clone --recursive https://github.com/matplotlib/matplotlib.git && \ + : + +# TODO: Come up with a different way to test matplotlib installation. +# Currently we will be running all the python files in galleries/tutorials +# But this directory structure might change in the future. In the past, "galleries/tutorials" +# didn't exist. Previously the repository just had a "tutorials" folder. +WORKDIR "matplotlib/galleries/tutorials" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_matplotlib_tests.sh . +RUN chmod +x run_matplotlib_tests.sh +# Run tests in run_matplotlib_tests.sh +CMD ["./run_matplotlib_tests.sh"] + diff --git a/test/test_artifacts/v1/numpy.test.Dockerfile b/test/test_artifacts/v1/numpy.test.Dockerfile new file mode 100644 index 00000000..d8fd071c --- /dev/null +++ b/test/test_artifacts/v1/numpy.test.Dockerfile @@ -0,0 +1,13 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +# Inorder to test numpy, we need pytest and hypothesis to be installed. +RUN micromamba install -y --freeze-installed conda-forge::pytest conda-forge::hypothesis +# Some unit tests in numpy requires gcc to be installed. +RUN sudo apt-get update && sudo apt-get install -y gcc +# Check https://numpy.org/doc/stable/reference/testing.html +# numpy.test() returns True if tests succeed else False. +# We need to flip the result so that we exit with status code as 0 if all the tests succeeded. +CMD ["python", "-c", "import numpy,sys; tests_succeeded = numpy.test(); sys.exit(not tests_succeeded)"] diff --git a/test/test_artifacts/v1/pandas.test.Dockerfile b/test/test_artifacts/v1/pandas.test.Dockerfile new file mode 100644 index 00000000..2ee57ced --- /dev/null +++ b/test/test_artifacts/v1/pandas.test.Dockerfile @@ -0,0 +1,8 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN micromamba install -y --freeze-installed -c conda-forge pytest hypothesis + +COPY --chown=$MAMBA_USER:$MAMBA_USER run_pandas_tests.py . +CMD ["python", "run_pandas_tests.py"] diff --git a/test/test_artifacts/v1/pytorch.examples.Dockerfile b/test/test_artifacts/v1/pytorch.examples.Dockerfile new file mode 100644 index 00000000..ca4e6615 --- /dev/null +++ b/test/test_artifacts/v1/pytorch.examples.Dockerfile @@ -0,0 +1,20 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN sudo apt-get update && \ + sudo apt-get install -y git && \ + git clone --recursive https://github.com/pytorch/examples && \ + : + +WORKDIR "examples" + +# There is a line in run_python_examples.sh which looks like: BASE_DIR=`pwd`"/"`dirname $0` +# When we run the shell script through /usr/local/bin/_entrypoint.sh, that line above doesn't work correctly. In our +# case, we properly set `pwd` to the directory that contains all the examples, so we just modify the script to change +# the previous line to look like: BASE_DIR=`pwd` +RUN sed -i 's/^BASE_DIR=.*pwd.*dirname.*/BASE_DIR=`pwd`/' run_python_examples.sh +RUN ./run_python_examples.sh install_deps + +# We skip `imagenet` because it requires a lot of resources and so aren't a good fit for us. +CMD ["./run_python_examples.sh", "dcgan,fast_neural_style,distributed,mnist,mnist_forward_forward,mnist_hogwild,mnist_rnn,regression,reinforcement_learning,siamese_network,super_resolution,time_sequence_prediction,vae,word_language_model,fx"] diff --git a/test/test_artifacts/v1/run_pandas_tests.py b/test/test_artifacts/v1/run_pandas_tests.py new file mode 100644 index 00000000..4cc8086c --- /dev/null +++ b/test/test_artifacts/v1/run_pandas_tests.py @@ -0,0 +1,20 @@ +import pandas, sys, os, site + +# We change the working directory here because there is at least one test (`test_html_template_extends_options`) which +# expects the directory to be 'pandas'. Ideally, we would have changed directories through a `WORKDIR` in Dockerfile +# but unfortunately it doesn't accept dynamic arguments. +site_packages_dir = site.getsitepackages()[0] +os.chdir(site_packages_dir) + +# pandas.test() by default runs with `-m "not slow and not network and not db"`. However, we found a few tests in the +# test_network.py file that should have been marked as "network" but weren't, so we skip those here. We skip S3 specific +# tests for the same reason. +# We skip `test_plain_axes` too: the Pandas dev environment expects matplotlib to be ">=3.6.1, <3.7.0" but the runtime +# expectation is just ">=3.6.1". Our image contains v3.7.1, so it meets the latter requirement but not the former. This +# particular test, however, only works with the former requirement. (We verified that the test succeeds if we manually +# drop the version to v3.6.x) So, we skip it. +tests_succeeded = pandas.test([ + '-m', '(not slow and not network and not db)', + '-k', '(not test_network and not s3 and not test_plain_axes)']) + +sys.exit(not tests_succeeded) diff --git a/test/test_artifacts/v1/scipy.test.Dockerfile b/test/test_artifacts/v1/scipy.test.Dockerfile new file mode 100644 index 00000000..106a0384 --- /dev/null +++ b/test/test_artifacts/v1/scipy.test.Dockerfile @@ -0,0 +1,12 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +# Inorder to test scipy, we need pytest and hypothesis to be installed. +RUN micromamba install -y --freeze-installed conda-forge::pytest conda-forge::hypothesis conda-forge::scipy-tests +# Check https://github.com/numpy/numpy/blob/main/doc/TESTS.rst +# Note: Testing guidelines are same for numpy and scipy. +# scipy.test() returns True if tests succeed else False. +# We need to flip the result so that we exit with status code as 0 if all the tests succeeded. +CMD ["python", "-c", "import scipy,sys; tests_succeeded = scipy.test(); sys.exit(not tests_succeeded)"] diff --git a/test/test_artifacts/v1/scripts/run_autogluon_tests.sh b/test/test_artifacts/v1/scripts/run_autogluon_tests.sh new file mode 100644 index 00000000..036d1432 --- /dev/null +++ b/test/test_artifacts/v1/scripts/run_autogluon_tests.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +AUTOGLUON_VERSION=$(micromamba list | grep autogluon | tr -s ' ' | cut -d ' ' -f 3) +git checkout tags/v$AUTOGLUON_VERSION + +# Run autogluon quick start as end-to-end check +jupyter nbconvert --execute --to python docs/tutorials/tabular/tabular-quick-start.ipynb +jupyter nbconvert --execute --to python docs/tutorials/timeseries/forecasting-quick-start.ipynb + +# Detect gpu and run multimodal quick start if presented +python -c "import torch; exit(0) if torch.cuda.is_available() else exit(1)" +ret=$? + +if [ $ret -eq 0 ] +then + jupyter nbconvert --execute --to python docs/tutorials/multimodal/multimodal_prediction/multimodal-quick-start.ipynb +fi diff --git a/test/test_artifacts/v1/scripts/run_boto3_tests.sh b/test/test_artifacts/v1/scripts/run_boto3_tests.sh new file mode 100644 index 00000000..ab0c7803 --- /dev/null +++ b/test/test_artifacts/v1/scripts/run_boto3_tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# We need to checkout the version of boto3 that is installed in the mamba environment. + +boto3_version=$(micromamba list | grep boto3 | tr -s ' ' | cut -d ' ' -f 3) +# Checkout the corresponding boto3 version +git checkout tags/$boto3_version + +# Run the unit and functional tests +pytest tests/unit tests/functional || exit $? diff --git a/test/test_artifacts/v1/scripts/run_keras_tests.sh b/test/test_artifacts/v1/scripts/run_keras_tests.sh new file mode 100644 index 00000000..aa8f7fdd --- /dev/null +++ b/test/test_artifacts/v1/scripts/run_keras_tests.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Ref: https://keras.io/guides/, https://github.com/keras-team/keras-io/tree/master + +for file in *.py; do + if [ "$file" != "transfer_learning.py" ]; then + # skipping transfer_learning.py because it has 20 epochs and it takes a very long time to execute + # https://github.com/keras-team/keras-io/blob/master/guides/transfer_learning.py#L562 + python "$file" || exit $? + fi +done diff --git a/test/test_artifacts/v1/scripts/run_matplotlib_tests.sh b/test/test_artifacts/v1/scripts/run_matplotlib_tests.sh new file mode 100644 index 00000000..13aa3fb3 --- /dev/null +++ b/test/test_artifacts/v1/scripts/run_matplotlib_tests.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Run all the tutorials +for file in *.py; do + python "$file" || exit $? +done + + diff --git a/test/test_artifacts/v1/scripts/run_pysdk_tests.sh b/test/test_artifacts/v1/scripts/run_pysdk_tests.sh new file mode 100644 index 00000000..2f49f122 --- /dev/null +++ b/test/test_artifacts/v1/scripts/run_pysdk_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# We need to checkout the version of sagemaker-python-sdk that is installed in the mamba environment. + +pysdk_version=$(micromamba list | grep sagemaker-python-sdk | tr -s ' ' | cut -d ' ' -f 3) +# Checkout the corresponding sagemaker-python-sdk version +git checkout tags/v$pysdk_version + +# Install test dependencies of sagemaker-python-sdk +# Using pip as some of the packages are not available on conda-forge +pip install -r requirements/extras/test_requirements.txt + +# Run the unit tests, ignoring tests which require AWS Configuration +# TODO: Re-evaluate the ignored tests since we are setting the AWS_DEFAULT_REGION as part of the Dockerfile. +pytest tests/unit --ignore=tests/unit/sagemaker/feature_store/ --ignore=tests/unit/sagemaker/jumpstart/ --ignore=tests/unit/sagemaker/workflow/ \ + --ignore=tests/unit/sagemaker/async_inference --ignore=tests/unit/test_model_card.py --ignore=tests/unit/test_model_card.py --ignore=tests/unit/test_processing.py \ + --ignore=tests/unit/test_tensorboard.py --ignore=tests/unit/sagemaker/async_inference --ignore=tests/unit/sagemaker/experiments --ignore tests/unit/sagemaker/local \ + --ignore tests/unit/sagemaker/monitor/test_data_capture_config.py --ignore tests/unit/sagemaker/experiments --ignore tests/unit/sagemaker/remote_function \ + --ignore tests/unit/sagemaker/model/test_deploy.py --deselect tests/unit/test_estimator.py::test_insert_invalid_source_code_args \ + --deselect tests/unit/sagemaker/tensorflow/test_estimator.py::test_insert_invalid_source_code_args || exit $? diff --git a/test/test_artifacts/v1/sm-python-sdk.test.Dockerfile b/test/test_artifacts/v1/sm-python-sdk.test.Dockerfile new file mode 100644 index 00000000..4a9c8d7d --- /dev/null +++ b/test/test_artifacts/v1/sm-python-sdk.test.Dockerfile @@ -0,0 +1,13 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN sudo apt-get update && sudo apt-get install -y git && \ + git clone --recursive https://github.com/aws/sagemaker-python-sdk.git && \ + : +# Sagemaker Python SDK's unit tests requires AWS_DEFAULT_REGION to be set. So, using an arbitrary value of us-east-1 +ENV AWS_DEFAULT_REGION=us-east-1 +WORKDIR "sagemaker-python-sdk" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_pysdk_tests.sh . +RUN chmod +x run_pysdk_tests.sh +CMD ["./run_pysdk_tests.sh"] diff --git a/test/test_artifacts/v1/tensorflow.examples.Dockerfile b/test/test_artifacts/v1/tensorflow.examples.Dockerfile new file mode 100644 index 00000000..f88bc9f2 --- /dev/null +++ b/test/test_artifacts/v1/tensorflow.examples.Dockerfile @@ -0,0 +1,16 @@ +ARG COSMOS_IMAGE +FROM $COSMOS_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +RUN sudo apt-get update && \ + sudo apt-get install -y git && \ + git clone --recursive https://github.com/tensorflow/docs.git && \ + : + +WORKDIR "docs/site/en/guide" +COPY --chown=$MAMBA_USER:$MAMBA_USER tensorflow ./ +RUN chmod +x run_tensorflow_example_notebooks.sh + +RUN micromamba install -y --freeze-installed -c conda-forge papermill + +CMD ["./run_tensorflow_example_notebooks.sh"] diff --git a/test/test_artifacts/v1/tensorflow/run_tensorflow_example_notebooks.sh b/test/test_artifacts/v1/tensorflow/run_tensorflow_example_notebooks.sh new file mode 100644 index 00000000..22887459 --- /dev/null +++ b/test/test_artifacts/v1/tensorflow/run_tensorflow_example_notebooks.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Create an empty notebook file for papermill's output +touch nb_output.ipynb + +# List of all referenced notebook files in Basics, Core, and In Depth sections of Tensorflow docs, excluding experimentals. +# https://www.tensorflow.org/guide +example_notebooks=('basics.ipynb' + 'tensor.ipynb' + 'variable.ipynb' + 'autodiff.ipynb' + 'intro_to_graphs.ipynb' + 'intro_to_modules.ipynb' + 'basic_training_loops.ipynb' + 'core/quickstart_core.ipynb' + 'core/logistic_regression_core.ipynb' + 'core/mlp_core.ipynb' + 'core/matrix_core.ipynb' + 'core/optimizers_core.ipynb' + 'tensor_slicing.ipynb' + 'advanced_autodiff.ipynb' + 'ragged_tensor.ipynb' + 'sparse_tensor.ipynb' + 'random_numbers.ipynb' +) + +for nb in ${example_notebooks[@]}; do + papermill $nb 'nb_output.ipynb' +done diff --git a/test/test_dockerfile_based_harness.py b/test/test_dockerfile_based_harness.py index fb9b937d..c0207b4b 100644 --- a/test/test_dockerfile_based_harness.py +++ b/test/test_dockerfile_based_harness.py @@ -26,7 +26,8 @@ ("pandas.test.Dockerfile", ['pandas']), ("sm-python-sdk.test.Dockerfile", ['sagemaker-python-sdk']), ("pytorch.examples.Dockerfile", ['pytorch']), - ("tensorflow.examples.Dockerfile", ['tensorflow'])]) + ("tensorflow.examples.Dockerfile", ['tensorflow']), + ("jupyter-ai.test.Dockerfile", ['jupyter-ai'])]) def test_dockerfiles_for_cpu(dockerfile_path: str, required_packages: List[str], local_image_version: str, use_gpu: bool): _validate_docker_images(dockerfile_path, required_packages, local_image_version, use_gpu, 'cpu') @@ -43,7 +44,8 @@ def test_dockerfiles_for_cpu(dockerfile_path: str, required_packages: List[str], ("pandas.test.Dockerfile", ['pandas']), ("sm-python-sdk.test.Dockerfile", ['sagemaker-python-sdk']), ("pytorch.examples.Dockerfile", ['pytorch']), - ("tensorflow.examples.Dockerfile", ['tensorflow'])]) + ("tensorflow.examples.Dockerfile", ['tensorflow']), + ("jupyter-ai.test.Dockerfile", ['jupyter-ai'])]) def test_dockerfiles_for_gpu(dockerfile_path: str, required_packages: List[str], local_image_version: str, use_gpu: bool): _validate_docker_images(dockerfile_path, required_packages, local_image_version, use_gpu, 'gpu')