Skbuild core

.github/workflows/cibuildwheels.yml

@@ -0,0 +1,109 @@
+name: Python wheels
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - main
+
+
+jobs:
+
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }} (${{ matrix.arch }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        arch: [x86_64, aarch64, arm64]
+        exclude:
+          - os: windows-latest
+            arch: aarch64
+          - os: macos-latest
+            arch: aarch64
+          - os: windows-latest
+            arch: arm64
+          - os: ubuntu-latest
+            arch: arm64
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Set up QEMU
+        if: ${{ matrix.arch == 'aarch64' }}
+        uses: docker/setup-qemu-action@v2
+
+      - name: Install Ninja
+        uses: seanmiddleditch/gha-setup-ninja@master
+
+      - name: Install MSVC amd64
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: amd64
+
+      - name: Build wheels (Windows)
+        if: runner.os == 'Windows'
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: 'cp38-win_amd64 cp39-win_amd64 cp310-win_amd64 cp311-win_amd64'
+          CIBW_BEFORE_TEST: pip install blosc2
+          CIBW_TEST_COMMAND: cmd /V /C "set "BLOSC_TRACE=1" && python {project}/examples/array_roundtrip.py"
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Build wheels (Mac OSX arm64)
+        if: ${{ matrix.os == 'macos-latest' && matrix.arch == 'arm64' }}
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: 'cp38-* cp39-* cp310-* cp311-*'
+          CIBW_BEFORE_TEST: pip install blosc2
+          CIBW_TEST_COMMAND: BLOSC_TRACE=1 python {project}/examples/array_roundtrip.py
+          CIBW_BUILD_VERBOSITY: 1
+          CIBW_ARCHS_MACOS: "arm64"
+
+      - name: Build wheels (Linux / Mac OSX)
+        if: ${{ matrix.os != 'windows-latest' && (matrix.arch == 'x86_64' || matrix.arch == 'aarch64') }}
+        uses: pypa/[email protected]
+        env:
+          CIBW_BEFORE_BUILD: python -m pip install --upgrade pip && python -m pip install -r requirements-build.txt
+          CIBW_BUILD: 'cp38-* cp39-* cp310-* cp311-*'
+          CIBW_SKIP: '*-manylinux*_i686 *-musllinux_* ${{ env.CIBW_SKIP}}'
+          CIBW_ARCHS_LINUX: ${{ matrix.arch }}
+          CIBW_BEFORE_TEST: pip install blosc2
+          CIBW_TEST_COMMAND: BLOSC_TRACE=1 python {project}/examples/array_roundtrip.py
+          CIBW_BUILD_VERBOSITY: 1
+          CIBW_ARCHS_MACOS: "x86_64"
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+
+  upload_pypi:
+    needs: [ build_wheels ]  # last but not least
+    runs-on: ubuntu-latest
+    # Only upload wheels when tagging (typically a release)
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.blosc_pypi_secret }}

CMakeLists.txt

@@ -13,32 +13,26 @@ set(CMAKE_CXX_STANDARD 20)
 message("Building Blosc2 plugin example...")
 
 # Find blosc2.h
-find_package(Python COMPONENTS Interpreter)
-if(Python_Interpreter_FOUND)
-    message(STATUS "Executable: ${Python_EXECUTABLE}")
-    IF (WIN32)
-        cmake_path(SET Python_ROOT NORMALIZE "${Python_EXECUTABLE}/..")
-    ELSE()
-        cmake_path(SET Python_ROOT NORMALIZE "${Python_EXECUTABLE}/../..")
-    ENDIF()
-    cmake_path(SET Python_INCLUDE NORMALIZE "${Python_ROOT}/include")
-    message(STATUS "Found Python include: ${Python_INCLUDE}")
-    cmake_path(SET Python_LIB NORMALIZE "${Python_ROOT}/lib")
-    message(STATUS "Found Python lib: ${Python_LIB}")
-    cmake_path(SET Python_LIB64 NORMALIZE "${Python_ROOT}/lib64")
-    message(STATUS "Found Python lib64: ${Python_LIB64}")
-    cmake_path(SET Python_Blosc2_INCLUDE NORMALIZE "${Python_INCLUDE}/blosc2.h")
-    cmake_path(HAS_FILENAME Python_Blosc2_INCLUDE HAS_BLOSC2)
-    if(HAS_BLOSC2)
-        set(BLOSC2_INCLUDE_DIR ${Python_INCLUDE})
-        message(STATUS "Found Blosc2 include: ${Python_Blosc2_INCLUDE}")
-    else()
-        message(FATAL_ERROR "No Blosc2 includes found.  Aborting.")
-    endif()
+find_package(Python COMPONENTS Interpreter NumPy Development.Module REQUIRED)
+message(STATUS "Executable: ${Python_EXECUTABLE}")
+message(STATUS "NumPy found: ${Python_NumPy_FOUND}")
+message(STATUS "NumPy version: ${Python_NumPy_VERSION}")
+message(STATUS "Python_NumPy_INCLUDE_DIRS: ${Python_NumPy_INCLUDE_DIRS}")
+if (UNIX)
+    cmake_path(SET Python_ROOT NORMALIZE "${Python_NumPy_INCLUDE_DIRS}/../../../../../..")
 else()
-    message(FATAL_ERROR "No Python found.  Aborting.")
+    cmake_path(SET Python_ROOT NORMALIZE "${Python_NumPy_INCLUDE_DIRS}/../../../../..")
+endif()
+cmake_path(SET Python_INCLUDE NORMALIZE "${Python_ROOT}/include")
+message(STATUS "Found Python include: ${Python_INCLUDE}")
+cmake_path(SET Python_Blosc2_INCLUDE NORMALIZE "${Python_ROOT}/include/blosc2.h")
+cmake_path(HAS_FILENAME Python_Blosc2_INCLUDE HAS_BLOSC2)
+if(HAS_BLOSC2)
+    set(BLOSC2_INCLUDE_DIR ${Python_INCLUDE})
+    message(STATUS "Found Blosc2 include: ${Python_Blosc2_INCLUDE}")
+else()
+    message(FATAL_ERROR "No Blosc2 includes found.  Aborting.")
 endif()
 
 include_directories("${BLOSC2_INCLUDE_DIR}")
 add_subdirectory(src)
-add_subdirectory(examples)

README.md

@@ -4,8 +4,22 @@ This is an example on how to create a wheel of a plugin for Blosc2.
 
 ## Create the wheel
 
+### For Linux
+
+```shell
+python -m cibuildwheel --only 'cp311-manylinux_x86_64'
+```
+
+### For Mac
+
 ```shell
-python setup.py bdist_wheel
+python -m cibuildwheel --only 'cp311-macosx_x86_64'
+```
+
+### For Windows
+
+```shell
+python -m cibuildwheel --only 'cp311-win_amd64'
 ```
 
 ## Verify the wheel is working
@@ -33,34 +47,16 @@ drwxr-xr-x  3 faltet  staff    96B Mar  6 13:45 __pycache__/
 To test that the wheel has been installed and works properly.
 
 ```shell
-cd _skbuild/*/cmake-build/examples/
-./test_plugin
-Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$)
-Compression ratio: 381.5 MB -> 2.9 MB (130.8x)
-Compression time: 0.265 s, 1437.0 MB/s
-Decompression time: 0.152 s, 2506.6 MB/s
-Successful roundtrip data <-> schunk !
+cd examples/
+python schunk_roundtrip.py
+Successful roundtrip!
 ```
 
 If you want to see that the plugin example filter its being applied
 instead of any other use `BLOSC_TRACE=1`.
 
-```shell
-BLOSC_TRACE=1 ./test_plugin
-Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$)
-[info] - libpath for plugin blosc2_plugin_example: /Users/martaiborra/miniconda3/envs/plugin_example/lib/python3.11/site-packages/blosc2_plugin_example/libblosc2_plugin_example.so
- (/Users/runner/work/python-blosc2/python-blosc2/blosc2/c-blosc2/blosc/blosc-private.h:265)
-Inside plugin_example forward function
-          ...
-Compression ratio: 381.5 MB -> 2.9 MB (130.8x)
-Compression time: 0.25 s, 1523.0 MB/s
-Inside plugin_example backward function
-          ...
-Decompression time: 0.156 s, 2444.7 MB/s
-Inside plugin_example backward function
-          ...
-Successful roundtrip data <-> schunk !
-```
+There is also a C example that can be compiled and run following the instructions in its `test_plugin.c` 
+file.
 
 In the future, you should be able to test that the wheel is working with this command:
 

blosc2_plugin_example/__init__.py

@@ -6,7 +6,6 @@
 #
 # See LICENSE.txt for details about copyright and rights to use.
 
-VERSION = 0.1
 
 import os
 import platform
@@ -18,7 +17,7 @@ def print_libpath():
     if system in ["Linux", "Darwin"]:
         libname = "libblosc2_plugin_example.so"
     elif system == "Windows":
-        libname = "libblosc2_plugin_example.dll"
+        libname = "blosc2_plugin_example.dll"
     else:
         raise RuntimeError("Unsupported system: ", system)
     libpath = os.path.abspath(Path(__file__).parent / libname)

examples/array_roundtrip.py

@@ -0,0 +1,27 @@
+#######################################################################
+# Copyright (c) 2019-present, Blosc Development Team <[email protected]>
+# All rights reserved.
+#
+# This source code is licensed under a BSD-style license (found in the
+# LICENSE file in the root directory of this source tree)
+#######################################################################
+
+
+import blosc2
+import numpy as np
+
+
+# Register the plugin_example
+blosc2.register_filter(250, None, None, "plugin_example")
+
+# Create blosc2 NDArray from a NumPy array
+shape = [100, 100]
+size = int(np.prod(shape))
+nparray = np.arange(size, dtype=np.int32).reshape(shape)
+blosc2_array = blosc2.asarray(nparray, cparams={"filters": [250]})
+
+# Retrieve data and check it is the same
+if np.array_equal(blosc2_array[...], nparray):
+    print("Successful roundtrip!")
+else:
+    print("Bad roundtrip! Try setting BLOSC_TRACE=1 envvar for more info on failure.")

examples/schunk_roundtrip.py

@@ -0,0 +1,31 @@
+#######################################################################
+# Copyright (c) 2019-present, Blosc Development Team <[email protected]>
+# All rights reserved.
+#
+# This source code is licensed under a BSD-style license (found in the
+# LICENSE file in the root directory of this source tree)
+#######################################################################
+
+import numpy as np
+
+import blosc2
+
+blosc2.register_filter(250, None, None, "plugin_example")
+
+# Set the compression and decompression parameters
+storage = {"contiguous": True, "cparams": {"filters": [250], "typesize": 4}}
+
+# Create the SChunk
+nchunks = 10
+data = np.arange(200 * 1000 * nchunks, dtype=np.int32)
+schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, data=data, **storage)
+
+cframe = schunk.to_cframe()
+
+schunk2 = blosc2.schunk_from_cframe(cframe, False)
+data2 = np.empty(data.shape, dtype=data.dtype)
+schunk2.get_slice(out=data2)
+if np.array_equal(data, data2):
+    print("Successful roundtrip!")
+else:
+    print("Bad roundtrip! Try setting BLOSC_TRACE=1 envvar for more info on failure.")

examples/test_plugin.c

@@ -7,6 +7,39 @@
    Since it won't be registered yet as a filter/codec in Blosc2, it will
    be registered as user-defined. So it's id must be between 160-255.
 
+   To compile and run this test:
+   cd examples
+   gcc test_plugin.c -o test_plugin -I /Users/martaiborra/miniforge3/envs/plugin_example_arm64/include \
+   -L /Users/martaiborra/miniforge3/envs/plugin_example_arm64/lib -lblosc2
+   # For MacOS
+   DYLD_LIBRARY_PATH=/Users/martaiborra/miniforge3/envs/plugin_example_arm64/lib/ ./test_plugin
+   # For linux
+   LD_LIBRARY_PATH=path_to_blosc2_lib ./test_plugin
+
+   To check that it is actually running the defined filter you can use BLOSC_TRACE=1.
+   BLOSC_TRACE=1 LD_LIBRARY_PATH=path_to_blosc2_lib ./test_plugin
+
+   Expected output:
+    Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$)
+    Compression ratio: 381.5 MB -> 2.9 MB (130.8x)
+    Compression time: 0.265 s, 1437.0 MB/s
+    Decompression time: 0.152 s, 2506.6 MB/s
+    Successful roundtrip data <-> schunk !
+
+   Expected output with BLOSC_TRACE=1:
+    Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$)
+    [info] - libpath for plugin blosc2_plugin_example: /Users/martaiborra/miniconda3/envs/plugin_example/lib/python3.11/site-packages/blosc2_plugin_example/libblosc2_plugin_example.so
+     (/Users/runner/work/python-blosc2/python-blosc2/blosc2/c-blosc2/blosc/blosc-private.h:265)
+    Inside plugin_example forward function
+              ...
+    Compression ratio: 381.5 MB -> 2.9 MB (130.8x)
+    Compression time: 0.25 s, 1523.0 MB/s
+    Inside plugin_example backward function
+              ...
+    Decompression time: 0.156 s, 2444.7 MB/s
+    Inside plugin_example backward function
+              ...
+    Successful roundtrip data <-> schunk !
  */