diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 25dd1cf40..748763812 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -55,6 +55,6 @@ jobs: path: _coverage - name: Upload coverage to codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: - files: '_coverage/coverage.info,_coverage/r_coverage.json' + files: '_coverage/coverage.info,_coverage/r_coverage.json,_coverage/python_coverage.xml' diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 85f12c2e7..c60f22901 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -36,7 +36,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 @@ -59,27 +59,6 @@ jobs: pytest python/tests -v -s - name: Run doctests - if: success() && matrix.python-version == '3.10' + if: success() && matrix.python-version == '3.12' run: | pytest --pyargs nanoarrow --doctest-modules - - - name: Coverage - if: success() && matrix.python-version == '3.10' - run: | - pip uninstall --yes nanoarrow - pip install pytest-cov Cython - pushd python - - # Build with Cython + gcc coverage options - pip install -e . - NANOARROW_PYTHON_COVERAGE=1 python setup.py build_ext --inplace - - # Run tests + coverage.py (generates .coverage + coverage.xml files) - python -m pytest --cov ./src/nanoarrow - python -m coverage xml - - - name: Upload coverage to codecov - if: success() && matrix.python-version == '3.10' - uses: codecov/codecov-action@v2 - with: - files: 'python/coverage.xml' diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index c5d12360f..b09439053 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -138,7 +138,9 @@ jobs: - { platform: "centos7", arch: "amd64", - compose_args: "-e NANOARROW_ACCEPT_IMPORT_GPG_KEYS_ERROR=1" + # Currently the Python on the centos7 image is 3.6, which does not support + # new enough setuptools to build the Python package. + compose_args: "-e NANOARROW_ACCEPT_IMPORT_GPG_KEYS_ERROR=1 -e TEST_PYTHON=0" } - { platform: "ubuntu", diff --git a/ci/scripts/coverage.sh b/ci/scripts/coverage.sh index ad7374abf..8c866bb94 100755 --- a/ci/scripts/coverage.sh +++ b/ci/scripts/coverage.sh @@ -52,7 +52,15 @@ case $# in ;; esac +maybe_activate_venv() { + if [ ! -z "${NANOARROW_PYTHON_VENV}" ]; then + source "${NANOARROW_PYTHON_VENV}/bin/activate" + fi +} + function main() { + maybe_activate_venv + SANDBOX_DIR="${TARGET_NANOARROW_DIR}/_coverage" if [ -d "${SANDBOX_DIR}" ]; then rm -rf "${SANDBOX_DIR}" @@ -107,6 +115,7 @@ function main() { --exclude "*/gtest/*" \ --exclude "*/flatcc/*" \ --exclude "*_generated.h" \ + --exclude "*nanoarrow/_deps/*" \ --output-file coverage.info # Generate the html coverage while we're here @@ -137,6 +146,35 @@ function main() { show_header "R package coverage summary" Rscript -e 'library(covr); print(readRDS("r_coverage.rds"))' popd + + # Build + test Python package with cython/gcc coverage options + show_header "Build + test Python package" + pushd "${SANDBOX_DIR}" + TARGET_NANOARROW_PYTHON_DIR="${TARGET_NANOARROW_DIR}/python" + + pushd "${TARGET_NANOARROW_PYTHON_DIR}" + NANOARROW_PYTHON_COVERAGE=1 python -m pip install -e . + + # Run tests + coverage.py (generates .coverage with absolute file paths) + python -m pytest --cov ./src/nanoarrow + + # Generate HTML report (file paths not important since it's just for viewing) + python -m coverage html + mv htmlcov "${SANDBOX_DIR}/python_htmlcov" + + # Move .coverage to the root directory and generate coverage.xml + # (generates relative file paths from the root of the repo) + mv .coverage .. + cp .coveragerc .. + pushd .. + python -m coverage xml + mv coverage.xml "${SANDBOX_DIR}/python_coverage.xml" + mv .coverage "${SANDBOX_DIR}/python_coverage.db" + rm .coveragerc + popd + + popd + popd } main diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 250bbe64c..84013602c 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -297,6 +297,59 @@ test_r() { popd } +activate_or_create_venv() { + if [ ! -z "${NANOARROW_PYTHON_VENV}" ]; then + show_info "Activating virtual environment at ${NANOARROW_PYTHON_VENV}" + # bash on Windows needs venv/Scripts/activate instead of venv/bin/activate + source "${NANOARROW_PYTHON_VENV}/bin/activate" || source "${NANOARROW_PYTHON_VENV}/Scripts/activate" + else + # Try python3 first, then try regular python (e.g., Windows) + if [ -z "${PYTHON_BIN}" ] && python3 --version >/dev/null; then + PYTHON_BIN=python3 + elif [ -z "${PYTHON_BIN}" ]; then + PYTHON_BIN=python + fi + + show_info "Creating temporary virtual environment using ${PYTHON_BIN}..." + "${PYTHON_BIN}" -m venv "${NANOARROW_TMPDIR}/venv" + # bash on Windows needs venv/Scripts/activate instead of venv/bin/activate + source "${NANOARROW_TMPDIR}/venv/bin/activate" || source "${NANOARROW_TMPDIR}/venv/Scripts/activate" + python -m pip install --upgrade pip + fi +} + +test_python() { + show_header "Build and test Python package" + activate_or_create_venv + + show_info "Installing build utilities" + python -m pip install --upgrade build + + pushd "${NANOARROW_SOURCE_DIR}/python" + + show_info "Building Python package" + rm -rf "${NANOARROW_TMPDIR}/python" + python -m build --wheel --outdir "${NANOARROW_TMPDIR}/python" + PYTHON_WHEEL_NAME=$(ls "${NANOARROW_TMPDIR}/python" | grep -e ".whl") + + # On Windows bash, pip install needs a Windows-style path + if uname | grep -e "_NT-" >/dev/null; then + pushd "${NANOARROW_TMPDIR}" + PYTHON_WHEEL_PATH="$(pwd -W)/python/${PYTHON_WHEEL_NAME}" + popd + else + PYTHON_WHEEL_PATH="${NANOARROW_TMPDIR}/python/${PYTHON_WHEEL_NAME}" + fi + + show_info "Installing Python package" + python -m pip install --force-reinstall "${PYTHON_WHEEL_PATH}[verify]" + + show_info "Testing wheel" + python -m pytest -vv + + popd +} + ensure_source_directory() { show_header "Ensuring source directory" @@ -346,6 +399,10 @@ test_source_distribution() { test_r fi + if [ ${TEST_PYTHON} -gt 0 ]; then + test_python + fi + popd } @@ -359,6 +416,7 @@ test_source_distribution() { : ${TEST_C:=${TEST_SOURCE}} : ${TEST_C_BUNDLED:=${TEST_C}} : ${TEST_R:=${TEST_SOURCE}} +: ${TEST_PYTHON:=${TEST_SOURCE}} TEST_SUCCESS=no diff --git a/python/bootstrap.py b/python/bootstrap.py index b540058af..4fb2b0e91 100644 --- a/python/bootstrap.py +++ b/python/bootstrap.py @@ -186,7 +186,10 @@ def copy_or_generate_nanoarrow_c(): is_in_nanoarrow_repo = "nanoarrow.h" in os.listdir( os.path.join(source_dir, "src", "nanoarrow") ) - has_cmake = os.system("cmake --version") == 0 + cmake_bin = os.getenv("CMAKE_BIN") + if not cmake_bin: + cmake_bin = "cmake" + has_cmake = os.system(f"{cmake_bin} --version") == 0 with tempfile.TemporaryDirectory() as build_dir: if is_in_nanoarrow_repo: @@ -206,7 +209,7 @@ def copy_or_generate_nanoarrow_c(): try: subprocess.run( [ - "cmake", + cmake_bin, "-B", build_dir, "-S", @@ -217,7 +220,7 @@ def copy_or_generate_nanoarrow_c(): ) subprocess.run( [ - "cmake", + cmake_bin, "--install", build_dir, "--prefix", diff --git a/python/pyproject.toml b/python/pyproject.toml index 7850a09f8..e176be267 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -22,15 +22,19 @@ dynamic = ["version"] readme = "README.md" description = "Python bindings to the nanoarrow C library" authors = [{name = "Apache Arrow Developers", email = "dev@arrow.apache.org"}] +maintainers = [{name = "Apache Arrow Developers", email = "dev@arrow.apache.org"}] license = {text = "Apache-2.0"} requires-python = ">=3.8" [project.optional-dependencies] test = ["pyarrow", "pytest", "numpy"] +verify = ["pytest", "numpy"] [project.urls] -homepage = "https://arrow.apache.org" -repository = "https://github.com/apache/arrow-nanoarrow" +Homepage = "https://arrow.apache.org" +Repository = "https://github.com/apache/arrow-nanoarrow" +Issues = "https://github.com/apache/arrow-nanoarrow/issues" +Changelog = "https://github.com/apache/arrow-nanoarrow/blob/main/CHANGELOG.md" [build-system] requires = [ diff --git a/python/setup.py b/python/setup.py index 92f8abc9b..cdffda2ba 100644 --- a/python/setup.py +++ b/python/setup.py @@ -52,7 +52,7 @@ def get_version(pkg_path): # Set some extra flags for compiling with coverage support -if os.getenv("NANOARROW_COVERAGE") == "1": +if os.getenv("NANOARROW_PYTHON_COVERAGE") == "1": extra_compile_args = ["--coverage"] extra_link_args = ["--coverage"] extra_define_macros = [("CYTHON_TRACE", 1)] @@ -65,7 +65,6 @@ def get_version(pkg_path): extra_link_args = [] extra_define_macros = [] - setup( ext_modules=[ Extension( diff --git a/python/src/nanoarrow/_lib.pyx b/python/src/nanoarrow/_lib.pyx index f77067c93..54d7bbdc0 100644 --- a/python/src/nanoarrow/_lib.pyx +++ b/python/src/nanoarrow/_lib.pyx @@ -915,7 +915,7 @@ cdef class CBufferView: if format_const != NULL: snprintf(self._format, sizeof(self._format), "%s", format_const) else: - snprintf(self._format, sizeof(self._format), "%ds", self._element_size_bits // 8) + snprintf(self._format, sizeof(self._format), "%ds", (self._element_size_bits // 8)) def __getbuffer__(self, Py_buffer *buffer, int flags): if self._device.device_type != ARROW_DEVICE_CPU: diff --git a/python/src/nanoarrow/_static_version.py b/python/src/nanoarrow/_static_version.py index 53569eff2..33ecbc5c2 100644 --- a/python/src/nanoarrow/_static_version.py +++ b/python/src/nanoarrow/_static_version.py @@ -18,7 +18,7 @@ # This file is part of 'miniver': https://github.com/jbweston/miniver # Replaced by version-bumping scripts at release time -version = "0.4.0dev0" +version = "0.4.0.dev0" # These values are only set if the distribution was created with 'git archive' refnames = "$Format:%D$" diff --git a/python/tests/test_version.py b/python/tests/test_version.py index 2b080d64c..701019ccd 100644 --- a/python/tests/test_version.py +++ b/python/tests/test_version.py @@ -21,7 +21,7 @@ def test_version(): - re_py_version = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(dev[0-9+])?$") + re_py_version = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9+])?$") assert re_py_version.match(na.__version__) is not None