From 2accbfc6a1c27671cb376ad63f0f6d2cf1de176d Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Mon, 29 Apr 2024 13:01:22 -0700 Subject: [PATCH] [SNOW-1297195] Add tox to daily jenkins job, fix tox.ini. (#1442) There were a few issues which prevent the daily snowpark pandas API jenkins job to run successfully: - tox.ini had `MODIN_PYTEST_NO_COV_CMD` missing - the jenkins invocation script requires tox to be installed - the tox_install_cmd.sh was using sh, but should be bash and variables were not defined. this PR addresses these issues. --- scripts/jenkins_regress_snowpandas.sh | 3 +++ scripts/tox_install_cmd.sh | 5 ++++- src/snowflake/snowpark/modin/config/envvars.py | 4 +++- tox.ini | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/jenkins_regress_snowpandas.sh b/scripts/jenkins_regress_snowpandas.sh index c5f55709045..e43e4561400 100644 --- a/scripts/jenkins_regress_snowpandas.sh +++ b/scripts/jenkins_regress_snowpandas.sh @@ -11,5 +11,8 @@ set -euxo pipefail # decrypt profile gpg --quiet --batch --yes --decrypt --passphrase="$GPG_KEY" --output "tests/parameters.py" $@ +# Install tox, which is by default not present in the environment. +python -m pip install tox + # Run snowpandas tests python -m tox -c $WORKING_DIR -e snowparkpandasjenkins-modin diff --git a/scripts/tox_install_cmd.sh b/scripts/tox_install_cmd.sh index be1ea828aa4..3f98d9c9b94 100755 --- a/scripts/tox_install_cmd.sh +++ b/scripts/tox_install_cmd.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash set -e @@ -13,6 +13,9 @@ done echo "${pip_options[*]}" +# Default to empty, to ensure snowflake_path variable is defined. +snowflake_path=${snowflake_path:-""} + if [[ -z "${snowflake_path}" ]]; then echo "Using Python Connector from PyPI" python -m pip install -U ${pip_options[@]} diff --git a/src/snowflake/snowpark/modin/config/envvars.py b/src/snowflake/snowpark/modin/config/envvars.py index eb16c03061b..352aa6e2e30 100644 --- a/src/snowflake/snowpark/modin/config/envvars.py +++ b/src/snowflake/snowpark/modin/config/envvars.py @@ -884,7 +884,9 @@ def _check_vars() -> None: # pragma: no cover and issubclass(obj, EnvironmentVariable) and not obj.is_abstract } - valid_names.update(["MODIN_PYTEST_CMD", "MODIN_PYTEST_DAILY_CMD"]) + valid_names.update( + ["MODIN_PYTEST_CMD", "MODIN_PYTEST_DAILY_CMD", "MODIN_PYTEST_NO_COV_CMD"] + ) found_names = {name for name in os.environ if name.startswith("MODIN_")} unknown = found_names - valid_names deprecated: dict[str, DeprecationDescriptor] = { diff --git a/tox.ini b/tox.ini index dd413c10e6c..fa928daaca1 100644 --- a/tox.ini +++ b/tox.ini @@ -62,6 +62,7 @@ setenv = SNOW_1314507_WORKAROUND_RERUN_FLAGS = --reruns 2 --only-rerun "Insufficient resource during interleaved execution." MODIN_PYTEST_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources MODIN_PYTEST_DAILY_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_DAILY_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources + MODIN_PYTEST_NO_COV_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} --ignore=tests/resources passenv = AWS_ACCESS_KEY_ID