Skip to content

SNOW-1344848: Suppress dynamic pivot warnings #6187

SNOW-1344848: Suppress dynamic pivot warnings

SNOW-1344848: Suppress dynamic pivot warnings #6187

Workflow file for this run

name: precommit test
on:
push:
branches: [ main ]
pull_request:
branches: '**'
workflow_dispatch:
inputs:
logLevel:
default: warning
description: "Log level"
required: true
tags:
description: "Test scenario tags"
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
name: Check linting
runs-on: ubuntu-latest-64-cores
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Display Python version
run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Run fix_lint
run: python -m tox -e fix_lint
type_checking:
name: Type Checking
runs-on: ubuntu-latest-64-cores
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Display Python version
run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Run pyright on Selected Files
run: python -m tox -e pyright
build:
needs: lint
name: Build Wheel File
runs-on: ubuntu-latest-64-cores
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip wheel
- name: Generate wheel
run: python -m pip wheel -v -w dist --no-deps .
- name: Show wheels generated
run: ls -lh dist
- uses: actions/upload-artifact@v4
with:
name: wheel
path: dist/
test:
name: Test py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
python-version: ["3.9", "3.10", "3.11"]
cloud-provider: [aws, azure, gcp]
exclude:
# only run macos with aws py3.9 for doctest
- os: macos-latest
python-version: "3.10"
- os: macos-latest
python-version: "3.11"
- os: macos-latest
python-version: "3.9"
cloud-provider: azure
- os: macos-latest
python-version: "3.9"
cloud-provider: gcp
# only run ubuntu with py3.9 on aws and py3.10 on azure
- os: ubuntu-latest-64-cores
python-version: "3.11"
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: azure
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: gcp
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: aws
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: gcp
# only run windows with py3.10 on gcp
- os: windows-latest-64-cores
python-version: "3.9"
- os: windows-latest-64-cores
python-version: "3.10"
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: aws
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: azure
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
# we only run doctest on macos
- if: ${{ matrix.os == 'macos-latest' }}
name: Run doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
# do not run other tests for macos
- if: ${{ matrix.os != 'macos-latest' }}
name: Run tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
path: |
.tox/.coverage
.tox/coverage.xml
test-local-testing:
name: Test Local Testing Module py-${{ matrix.os.download_name }}-${{ matrix.python-version }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos # it includes doctest
python-version: ["3.8", "3.9", "3.10", "3.11"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run tests
run: python -m tox -e "py${PYTHON_VERSION/\./}-local"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-local-testing
path: |
.tox/.coverage
.tox/coverage.xml
test-enable-cte-optimization:
name: Test Enable CTE Optimization py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest-64-cores]
python-version: ["3.9"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --enable_cte_optimization
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization
path: |
.tox/.coverage
.tox/coverage.xml
test-snowpark-pandas:
name: Test modin-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
python-version: [ "3.9", "3.10", "3.11" ]
cloud-provider: [aws, azure, gcp]
exclude:
# only run macos with aws py3.9 for doctest
- os: macos-latest
python-version: "3.10"
- os: macos-latest
python-version: "3.11"
- os: macos-latest
python-version: "3.9"
cloud-provider: azure
- os: macos-latest
python-version: "3.9"
cloud-provider: gcp
# only run ubuntu with py3.9 on aws and py3.10 on azure
- os: ubuntu-latest-64-cores
python-version: "3.11"
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: azure
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: gcp
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: aws
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: gcp
# only run windows with py3.10 on gcp
- os: windows-latest-64-cores
python-version: "3.9"
- os: windows-latest-64-cores
python-version: "3.10"
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: aws
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: azure
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
# only run doctest for macos on aws
- if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }}
name: Run Snowpark pandas API doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
# do not run other tests for macos on aws
- if: ${{ !(matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws') }}
name: Run Snowpark pandas API tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasnotdoctest-modin-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-pandas-testing
path: |
.tox/.coverage
.tox/coverage.xml
test-modin-extra-without-pandas-extra:
name: Test modin extra without pandas extra py-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}.
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos
python-version: ["3.9"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run all tests
run: python -m tox -e "modin_extra_without_pandas_extra"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
- uses: actions/upload-artifact@v4
with:
name: coverage_modin_extra_without_pandas_extra
path: |
.tox/.coverage
.tox/coverage.xml
combine-coverage:
if: ${{ success() || failure() }}
name: Combine coverage
needs: test
runs-on: ubuntu-latest-64-cores
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
path: artifacts
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Collect all coverages to one dir
run: |
python -c '
from pathlib import Path
import shutil
src_dir = Path("artifacts")
dst_dir = Path(".") / ".tox"
dst_dir.mkdir()
for src_file in src_dir.glob("*/.coverage"):
dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
print("{} copy to {}".format(src_file, dst_file))
shutil.copy(str(src_file), str(dst_file))'
- name: Combine coverages
run: python -m tox -e coverage
- name: Publish html coverage
uses: actions/upload-artifact@v4
with:
name: overall_cov_html
path: .tox/htmlcov
- name: Publish xml coverage
uses: actions/upload-artifact@v4
with:
name: overall_cov_xml
path: .tox/coverage.xml
- uses: codecov/codecov-action@v1
with:
file: .tox/coverage.xml
doc:
needs: lint
name: Build Doc
runs-on: ubuntu-latest-64-cores
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install Snowpark
run: python -m pip install ".[modin-development]"
- name: Install Sphinx
run: python -m pip install sphinx
- name: Build document
working-directory: docs
# treat warning as failure but complete the entire process
run: |
make clean
make html SPHINXOPTS="-W --keep-going"
- name: Upload html files
uses: actions/upload-artifact@v4
with:
name: docs
path: docs/build/html