SNOW-1318080: local testing move use object logic into session.py for better sql query isolation #6188

Workflow file for this run

.github/workflows/precommit.yml at 7a5f559

	name: precommit test
	on:
	push:
	branches: [ main ]
	pull_request:
	branches: '**'
	workflow_dispatch:
	inputs:
	logLevel:
	default: warning
	description: "Log level"
	required: true
	tags:
	description: "Test scenario tags"

	concurrency:
	group: ${{ github.ref }}
	cancel-in-progress: true

	jobs:
	lint:
	name: Check linting
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.9'
	- name: Display Python version
	run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
	- name: Upgrade setuptools and pip
	run: python -m pip install -U setuptools pip
	- name: Install tox
	run: python -m pip install tox
	- name: Run fix_lint
	run: python -m tox -e fix_lint

	type_checking:
	name: Type Checking
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.9'
	- name: Display Python version
	run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
	- name: Upgrade setuptools and pip
	run: python -m pip install -U setuptools pip
	- name: Install tox
	run: python -m pip install tox
	- name: Run pyright on Selected Files
	run: python -m tox -e pyright

	build:
	needs: lint
	name: Build Wheel File
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: 3.9
	- name: Upgrade setuptools and pip
	run: python -m pip install -U setuptools pip wheel
	- name: Generate wheel
	run: python -m pip wheel -v -w dist --no-deps .
	- name: Show wheels generated
	run: ls -lh dist
	- uses: actions/upload-artifact@v4
	with:
	name: wheel
	path: dist/

	test:
	name: Test py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
	needs: build
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
	python-version: ["3.9", "3.10", "3.11"]
	cloud-provider: [aws, azure, gcp]
	exclude:
	# only run macos with aws py3.9 for doctest
	- os: macos-latest
	python-version: "3.10"
	- os: macos-latest
	python-version: "3.11"
	- os: macos-latest
	python-version: "3.9"
	cloud-provider: azure
	- os: macos-latest
	python-version: "3.9"
	cloud-provider: gcp
	# only run ubuntu with py3.9 on aws and py3.10 on azure
	- os: ubuntu-latest-64-cores
	python-version: "3.11"
	- os: ubuntu-latest-64-cores
	python-version: "3.9"
	cloud-provider: azure
	- os: ubuntu-latest-64-cores
	python-version: "3.9"
	cloud-provider: gcp
	- os: ubuntu-latest-64-cores
	python-version: "3.10"
	cloud-provider: aws
	- os: ubuntu-latest-64-cores
	python-version: "3.10"
	cloud-provider: gcp
	# only run windows with py3.10 on gcp
	- os: windows-latest-64-cores
	python-version: "3.9"
	- os: windows-latest-64-cores
	python-version: "3.10"
	- os: windows-latest-64-cores
	python-version: "3.11"
	cloud-provider: aws
	- os: windows-latest-64-cores
	python-version: "3.11"
	cloud-provider: azure
	steps:
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Display Python version
	run: python -c "import sys; print(sys.version)"
	- name: Decrypt parameters.py
	shell: bash
	run: .github/scripts/decrypt_parameters.sh
	env:
	PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
	CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
	- name: Download wheel(s)
	uses: actions/download-artifact@v4
	with:
	name: wheel
	path: dist
	- name: Show wheels downloaded
	run: ls -lh dist
	shell: bash
	- name: Upgrade setuptools, pip and wheel
	run: python -m pip install -U setuptools pip wheel
	- name: Install tox
	run: python -m pip install tox
	# we only run doctest on macos
	- if: ${{ matrix.os == 'macos-latest' }}
	name: Run doctests
	run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	cloud_provider: ${{ matrix.cloud-provider }}
	PYTEST_ADDOPTS: --color=yes --tb=short
	TOX_PARALLEL_NO_SPINNER: 1
	# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
	# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
	shell: bash
	# do not run other tests for macos
	- if: ${{ matrix.os != 'macos-latest' }}
	name: Run tests (excluding doctests)
	run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	cloud_provider: ${{ matrix.cloud-provider }}
	PYTEST_ADDOPTS: --color=yes --tb=short
	TOX_PARALLEL_NO_SPINNER: 1
	shell: bash
	- name: Combine coverages
	run: python -m tox -e coverage --skip-missing-interpreters false
	shell: bash
	env:
	SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
	- uses: actions/upload-artifact@v4
	with:
	name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
	path: \|
	.tox/.coverage
	.tox/coverage.xml

	test-local-testing:
	name: Test Local Testing Module py-${{ matrix.os.download_name }}-${{ matrix.python-version }}
	needs: build
	runs-on: ${{ matrix.os.image_name }}
	strategy:
	fail-fast: false
	matrix:
	os:
	- image_name: macos-latest
	download_name: macos # it includes doctest
	python-version: ["3.8", "3.9", "3.10", "3.11"]
	cloud-provider: [aws]
	steps:
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Display Python version
	run: python -c "import sys; print(sys.version)"
	- name: Decrypt parameters.py
	shell: bash
	run: .github/scripts/decrypt_parameters.sh
	env:
	PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
	CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
	- name: Download wheel(s)
	uses: actions/download-artifact@v4
	with:
	name: wheel
	path: dist
	- name: Show wheels downloaded
	run: ls -lh dist
	shell: bash
	- name: Upgrade setuptools, pip and wheel
	run: python -m pip install -U setuptools pip wheel
	- name: Install tox
	run: python -m pip install tox
	- name: Run tests
	run: python -m tox -e "py${PYTHON_VERSION/\./}-local"
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	cloud_provider: ${{ matrix.cloud-provider }}
	PYTEST_ADDOPTS: --color=yes --tb=short
	TOX_PARALLEL_NO_SPINNER: 1
	SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
	shell: bash
	- name: Combine coverages
	run: python -m tox -e coverage --skip-missing-interpreters false
	shell: bash
	env:
	SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
	- uses: actions/upload-artifact@v4
	with:
	name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-local-testing
	path: \|
	.tox/.coverage
	.tox/coverage.xml

	test-enable-cte-optimization:
	name: Test Enable CTE Optimization py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
	needs: build
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest-64-cores]
	python-version: ["3.9"]
	cloud-provider: [aws]
	steps:
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Display Python version
	run: python -c "import sys; print(sys.version)"
	- name: Decrypt parameters.py
	shell: bash
	run: .github/scripts/decrypt_parameters.sh
	env:
	PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
	CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
	- name: Download wheel(s)
	uses: actions/download-artifact@v4
	with:
	name: wheel
	path: dist
	- name: Show wheels downloaded
	run: ls -lh dist
	shell: bash
	- name: Upgrade setuptools, pip and wheel
	run: python -m pip install -U setuptools pip wheel
	- name: Install tox
	run: python -m pip install tox
	- name: Run tests (excluding doctests)
	run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	cloud_provider: ${{ matrix.cloud-provider }}
	PYTEST_ADDOPTS: --color=yes --tb=short --enable_cte_optimization
	TOX_PARALLEL_NO_SPINNER: 1
	shell: bash
	- name: Combine coverages
	run: python -m tox -e coverage --skip-missing-interpreters false
	shell: bash
	env:
	SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
	- uses: actions/upload-artifact@v4
	with:
	name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization
	path: \|
	.tox/.coverage
	.tox/coverage.xml

	test-snowpark-pandas:
	name: Test modin-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
	needs: build
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
	python-version: [ "3.9", "3.10", "3.11" ]
	cloud-provider: [aws, azure, gcp]
	exclude:
	# only run macos with aws py3.9 for doctest
	- os: macos-latest
	python-version: "3.10"
	- os: macos-latest
	python-version: "3.11"
	- os: macos-latest
	python-version: "3.9"
	cloud-provider: azure
	- os: macos-latest
	python-version: "3.9"
	cloud-provider: gcp
	# only run ubuntu with py3.9 on aws and py3.10 on azure
	- os: ubuntu-latest-64-cores
	python-version: "3.11"
	- os: ubuntu-latest-64-cores
	python-version: "3.9"
	cloud-provider: azure
	- os: ubuntu-latest-64-cores
	python-version: "3.9"
	cloud-provider: gcp
	- os: ubuntu-latest-64-cores
	python-version: "3.10"
	cloud-provider: aws
	- os: ubuntu-latest-64-cores
	python-version: "3.10"
	cloud-provider: gcp
	# only run windows with py3.10 on gcp
	- os: windows-latest-64-cores
	python-version: "3.9"
	- os: windows-latest-64-cores
	python-version: "3.10"
	- os: windows-latest-64-cores
	python-version: "3.11"
	cloud-provider: aws
	- os: windows-latest-64-cores
	python-version: "3.11"
	cloud-provider: azure
	steps:
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Display Python version
	run: python -c "import sys; print(sys.version)"
	- name: Decrypt parameters.py
	shell: bash
	run: .github/scripts/decrypt_parameters.sh
	env:
	PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
	CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
	- name: Download wheel(s)
	uses: actions/download-artifact@v4
	with:
	name: wheel
	path: dist
	- name: Show wheels downloaded
	run: ls -lh dist
	shell: bash
	- name: Upgrade setuptools, pip and wheel
	run: python -m pip install -U setuptools pip wheel
	- name: Install tox
	run: python -m pip install tox
	# only run doctest for macos on aws
	- if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }}
	name: Run Snowpark pandas API doctests
	run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	cloud_provider: ${{ matrix.cloud-provider }}
	PYTEST_ADDOPTS: --color=yes --tb=short
	TOX_PARALLEL_NO_SPINNER: 1
	# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
	# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
	shell: bash
	# do not run other tests for macos on aws
	- if: ${{ !(matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws') }}
	name: Run Snowpark pandas API tests (excluding doctests)
	run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasnotdoctest-modin-ci"
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	cloud_provider: ${{ matrix.cloud-provider }}
	PYTEST_ADDOPTS: --color=yes --tb=short
	TOX_PARALLEL_NO_SPINNER: 1
	shell: bash
	- name: Combine coverages
	run: python -m tox -e coverage --skip-missing-interpreters false
	shell: bash
	env:
	SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
	- uses: actions/upload-artifact@v4
	with:
	name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-pandas-testing
	path: \|
	.tox/.coverage
	.tox/coverage.xml

	test-modin-extra-without-pandas-extra:
	name: Test modin extra without pandas extra py-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}.
	needs: build
	runs-on: ${{ matrix.os.image_name }}
	strategy:
	fail-fast: false
	matrix:
	os:
	- image_name: macos-latest
	download_name: macos
	python-version: ["3.9"]
	cloud-provider: [aws]
	steps:
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Display Python version
	run: python -c "import sys; print(sys.version)"
	- name: Decrypt parameters.py
	shell: bash
	run: .github/scripts/decrypt_parameters.sh
	env:
	PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
	CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
	- name: Download wheel(s)
	uses: actions/download-artifact@v4
	with:
	name: wheel
	path: dist
	- name: Show wheels downloaded
	run: ls -lh dist
	shell: bash
	- name: Upgrade setuptools, pip and wheel
	run: python -m pip install -U setuptools pip wheel
	- name: Install tox
	run: python -m pip install tox
	- name: Run all tests
	run: python -m tox -e "modin_extra_without_pandas_extra"
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	cloud_provider: ${{ matrix.cloud-provider }}
	PYTEST_ADDOPTS: --color=yes --tb=short
	TOX_PARALLEL_NO_SPINNER: 1
	shell: bash
	- name: Combine coverages
	run: python -m tox -e coverage --skip-missing-interpreters false
	shell: bash
	- uses: actions/upload-artifact@v4
	with:
	name: coverage_modin_extra_without_pandas_extra
	path: \|
	.tox/.coverage
	.tox/coverage.xml

	combine-coverage:
	if: ${{ success() \|\| failure() }}
	name: Combine coverage
	needs:
	- test
	- test-snowpark-pandas
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/download-artifact@v4
	with:
	path: artifacts
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.9'
	- name: Display Python version
	run: python -c "import sys; print(sys.version)"
	- name: Upgrade setuptools and pip
	run: python -m pip install -U setuptools pip
	- name: Install tox
	run: python -m pip install tox
	- name: Collect all coverages to one dir
	run: \|
	python -c '
	from pathlib import Path
	import shutil

	src_dir = Path("artifacts")
	dst_dir = Path(".") / ".tox"
	dst_dir.mkdir()
	for src_file in src_dir.glob("*/.coverage"):
	dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
	print("{} copy to {}".format(src_file, dst_file))
	shutil.copy(str(src_file), str(dst_file))'
	- name: Combine coverages
	run: python -m tox -e coverage
	- name: Publish html coverage
	uses: actions/upload-artifact@v4
	with:
	name: overall_cov_html
	path: .tox/htmlcov
	- name: Publish xml coverage
	uses: actions/upload-artifact@v4
	with:
	name: overall_cov_xml
	path: .tox/coverage.xml
	- uses: codecov/codecov-action@v1
	with:
	file: .tox/coverage.xml

	doc:
	needs: lint
	name: Build Doc
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.9'
	- name: Upgrade setuptools and pip
	run: python -m pip install -U setuptools pip
	- name: Install Snowpark
	run: python -m pip install ".[modin-development]"
	- name: Install Sphinx
	run: python -m pip install sphinx
	- name: Build document
	working-directory: docs
	# treat warning as failure but complete the entire process
	run: \|
	make clean
	make html SPHINXOPTS="-W --keep-going"
	- name: Upload html files
	uses: actions/upload-artifact@v4
	with:
	name: docs
	path: docs/build/html

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

SNOW-1318080: local testing move use object logic into session.py for better sql query isolation #6188

Workflow file

SNOW-1318080: local testing move use object logic into session.py for better sql query isolation #6188

Jobs

Run details

Workflow file for this run