Merge branch 'main' into jrose_snow_1269037_nat_support

snowflakedb · Apr 26, 2024 · 14c6be2 · 14c6be2
2 parents deac93d + d758658
commit 14c6be2
Show file tree

Hide file tree

Showing 382 changed files with 155,573 additions and 131 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1 +1,8 @@
 * @snowflakedb/snowpark-python-api-reviewers
+/src/snowflake/snowpark/modin/ @snowflakedb/snowpandas
+/tests/integ/modin/ @snowflakedb/snowpandas
+/tests/unit/modin/ @snowflakedb/snowpandas
+/.github/ @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers
+/scripts/ @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers
+setup.py @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers
+tox.ini @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers
diff --git a/.github/workflows/changedoc_snowpark_pandas.yml b/.github/workflows/changedoc_snowpark_pandas.yml
@@ -0,0 +1,22 @@
+name: Snowpark pandas Changedoc Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, labeled, unlabeled]
+    branches:
+      - pandas-main
+    paths:
+      - 'src/snowflake/snowpark/modin/**'
+
+jobs:
+  check_pandas_change_doc:
+    runs-on: ubuntu-latest
+    if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-PANDAS-CHANGEDOC-UPDATES')}}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Ensure Snowpark pandas docs is updated
+      run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q "docs/source/modin"
diff --git a/.github/workflows/changelog_snowpark_pandas.yml b/.github/workflows/changelog_snowpark_pandas.yml
@@ -0,0 +1,22 @@
+name: Snowpark pandas Changelog Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, labeled, unlabeled]
+    branches:
+      - pandas-main
+    paths:
+      - 'src/snowflake/snowpark/modin/**'
+
+jobs:
+  check_pandas_change_log:
+    runs-on: ubuntu-latest
+    if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-PANDAS-CHANGELOG-UPDATES')}}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Ensure PANDAS_CHANGELOG.md is updated
+      run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "src/snowflake/snowpark/modin/plugin/PANDAS_CHANGELOG.md"
diff --git a/.github/workflows/daily_modin_precommit.yml b/.github/workflows/daily_modin_precommit.yml
@@ -0,0 +1,301 @@
+# This is copied from original daily_precommit.yml with one change: only run Snowpark pandas tests
+
+name: Daily Snowpark pandas API test
+on:
+  schedule:
+    # 9 AM UTC
+    - cron: "0 9 * * *"
+  workflow_dispatch:
+      inputs:
+        logLevel:
+          default: warning
+          description: "Log level"
+          required: true
+        tags:
+          description: "Test scenario tags"
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Check linting
+    runs-on: ubuntu-latest-64-cores
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Display Python version
+        run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
+      - name: Upgrade setuptools and pip
+        run: python -m pip install -U setuptools pip
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Run fix_lint
+        run: python -m tox -e fix_lint
+
+  build:
+    needs: lint
+    name: Build Wheel File
+    runs-on: ubuntu-latest-64-cores
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Upgrade setuptools and pip
+        run: python -m pip install -U setuptools pip wheel
+      - name: Generate wheel
+        run: python -m pip wheel -v -w dist --no-deps .
+      - name: Show wheels generated
+        run: ls -lh dist
+      - uses: actions/upload-artifact@v1
+        with:
+          name: wheel
+          path: dist/
+
+  test-unsupported-py38:
+    name: Test importing Snowpark pandas with Python 3.8 fails
+    needs: build
+    runs-on: ubuntu-latest-64-cores
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Upgrade setuptools and pip
+        run: python -m pip install -U setuptools pip
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Ensure importing Snowpark pandas fails on Python 3.8
+        run: tox -e snowpark_pandas_py38_import_error
+
+  test:
+    name: Test modin-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+    needs: build
+    runs-on: ${{ matrix.os.image_name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - image_name: ubuntu-latest-64-cores
+            download_name: linux
+          - image_name: macos-latest
+            download_name: macos
+          - image_name: windows-latest-64-cores
+            download_name: windows
+        python-version: ["3.9", "3.10", "3.11"]
+        cloud-provider: [aws, azure, gcp]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Decrypt parameters.py
+        shell: bash
+        run: .github/scripts/decrypt_parameters.sh
+        env:
+          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
+          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Upgrade setuptools, pip and wheel
+        run: python -m pip install -U setuptools pip wheel
+      - name: Install tox
+        run: python -m pip install tox
+      - if: ${{ contains('macos', matrix.os.download_name) }}
+        name: Run Snowpark pandas API doctests
+        run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
+          # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
+        shell: bash
+      - name: Run Snowpark pandas API tests (excluding doctests)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasdailynotdoctest-modin-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+      - name: Combine coverages
+        run: python -m tox -e coverage --skip-missing-interpreters false
+        shell: bash
+        env:
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+      - uses: actions/upload-artifact@v3
+        with:
+          name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+          path: |
+            .tox/.coverage
+            .tox/coverage.xml
+
+  test-disable-sql-simplifier: # Will be removed after sql simplifier is stable and no option to opt out.
+    name: Test Disable SQL Simplifier modin-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+    needs: build
+    runs-on: ${{ matrix.os.image_name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - image_name: macos-latest
+            download_name: macos  # it includes doctest
+        python-version: ["3.9", "3.10", "3.11"]
+        cloud-provider: [aws]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Decrypt parameters.py
+        shell: bash
+        run: .github/scripts/decrypt_parameters.sh
+        env:
+          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
+          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Upgrade setuptools, pip and wheel
+        run: python -m pip install -U setuptools pip wheel
+      - name: Install tox
+        run: python -m pip install tox
+      - if: ${{ contains('macos', matrix.os.download_name) }}
+        name: Run Snowpark pandas API doctests
+        run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+      - name: Run Snowpark pandas API tests (excluding doctests)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasdailynotdoctest-modin-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+      - name: Combine coverages
+        run: python -m tox -e coverage --skip-missing-interpreters false
+        shell: bash
+        env:
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+      - uses: actions/upload-artifact@v3
+        with:
+          name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-disable-sql-simplifier
+          path: |
+            .tox/.coverage
+            .tox/coverage.xml
+
+  combine-coverage:
+    if: ${{ success() || failure() }}
+    name: Combine coverage
+    needs: test
+    runs-on: ubuntu-latest-64-cores
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Upgrade setuptools and pip
+        run: python -m pip install -U setuptools pip
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Collect all coverages to one dir
+        run: |
+          python -c '
+          from pathlib import Path
+          import shutil
+
+          src_dir = Path("artifacts")
+          dst_dir = Path(".") / ".tox"
+          dst_dir.mkdir()
+          for src_file in src_dir.glob("*/.coverage"):
+              dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
+              print("{} copy to {}".format(src_file, dst_file))
+              shutil.copy(str(src_file), str(dst_file))'
+      - name: Combine coverages
+        run: python -m tox -e coverage
+      - name: Publish html coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: overall_cov_html
+          path: .tox/htmlcov
+      - name: Publish xml coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: overall_cov_xml
+          path: .tox/coverage.xml
+      - uses: codecov/codecov-action@v1
+        with:
+          file: .tox/coverage.xml
+
+  doc:
+    needs: lint
+    name: Build Doc
+    runs-on: ubuntu-latest-64-cores
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Upgrade setuptools and pip
+        run: python -m pip install -U setuptools pip
+      - name: Install Snowpark
+        run: python -m pip install ".[modin-development]"
+      - name: Install Sphinx
+        run: python -m pip install sphinx
+      - name: Build document
+        working-directory: docs
+        # treat warning as failure but complete the entire process
+        run: |
+          make clean
+          make html SPHINXOPTS="-W --keep-going"
+      - name: Upload html files
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs
+          path: docs/build/html