diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml new file mode 100644 index 0000000..995dd7f --- /dev/null +++ b/.github/workflows/cibuildwheels.yml @@ -0,0 +1,109 @@ +name: Python wheels +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + tags: + - '*' + pull_request: + branches: + - main + + +jobs: + + build_wheels: + name: Build wheels on ${{ matrix.os }} (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + arch: [x86_64, aarch64, arm64] + exclude: + - os: windows-latest + arch: aarch64 + - os: macos-latest + arch: aarch64 + - os: windows-latest + arch: arm64 + - os: ubuntu-latest + arch: arm64 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Set up QEMU + if: ${{ matrix.arch == 'aarch64' }} + uses: docker/setup-qemu-action@v2 + + - name: Install Ninja + uses: seanmiddleditch/gha-setup-ninja@master + + - name: Install MSVC amd64 + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: amd64 + + - name: Build wheels (Windows) + if: runner.os == 'Windows' + uses: pypa/cibuildwheel@v2.14.1 + env: + CIBW_BUILD: 'cp38-win_amd64 cp39-win_amd64 cp310-win_amd64 cp311-win_amd64' + CIBW_BEFORE_TEST: pip install blosc2 + CIBW_TEST_COMMAND: cmd /V /C "set "BLOSC_TRACE=1" && python {project}/examples/array_roundtrip.py" + CIBW_BUILD_VERBOSITY: 1 + + - name: Build wheels (Mac OSX arm64) + if: ${{ matrix.os == 'macos-latest' && matrix.arch == 'arm64' }} + uses: pypa/cibuildwheel@v2.14.1 + env: + CIBW_BUILD: 'cp38-* cp39-* cp310-* cp311-*' + CIBW_BEFORE_TEST: pip install blosc2 + CIBW_TEST_COMMAND: BLOSC_TRACE=1 python {project}/examples/array_roundtrip.py + CIBW_BUILD_VERBOSITY: 1 + CIBW_ARCHS_MACOS: "arm64" + + - name: Build wheels (Linux / Mac OSX) + if: ${{ matrix.os != 'windows-latest' && (matrix.arch == 'x86_64' || matrix.arch == 'aarch64') }} + uses: pypa/cibuildwheel@v2.14.1 + env: + CIBW_BEFORE_BUILD: python -m pip install --upgrade pip && python -m pip install -r requirements-build.txt + CIBW_BUILD: 'cp38-* cp39-* cp310-* cp311-*' + CIBW_SKIP: '*-manylinux*_i686 *-musllinux_* ${{ env.CIBW_SKIP}}' + CIBW_ARCHS_LINUX: ${{ matrix.arch }} + CIBW_BEFORE_TEST: pip install blosc2 + CIBW_TEST_COMMAND: BLOSC_TRACE=1 python {project}/examples/array_roundtrip.py + CIBW_BUILD_VERBOSITY: 1 + CIBW_ARCHS_MACOS: "x86_64" + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + + upload_pypi: + needs: [ build_wheels ] # last but not least + runs-on: ubuntu-latest + # Only upload wheels when tagging (typically a release) + if: startsWith(github.event.ref, 'refs/tags') + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.blosc_pypi_secret }} diff --git a/CMakeLists.txt b/CMakeLists.txt index ea05d44..81fffa8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,32 +13,26 @@ set(CMAKE_CXX_STANDARD 20) message("Building Blosc2 plugin example...") # Find blosc2.h -find_package(Python COMPONENTS Interpreter) -if(Python_Interpreter_FOUND) - message(STATUS "Executable: ${Python_EXECUTABLE}") - IF (WIN32) - cmake_path(SET Python_ROOT NORMALIZE "${Python_EXECUTABLE}/..") - ELSE() - cmake_path(SET Python_ROOT NORMALIZE "${Python_EXECUTABLE}/../..") - ENDIF() - cmake_path(SET Python_INCLUDE NORMALIZE "${Python_ROOT}/include") - message(STATUS "Found Python include: ${Python_INCLUDE}") - cmake_path(SET Python_LIB NORMALIZE "${Python_ROOT}/lib") - message(STATUS "Found Python lib: ${Python_LIB}") - cmake_path(SET Python_LIB64 NORMALIZE "${Python_ROOT}/lib64") - message(STATUS "Found Python lib64: ${Python_LIB64}") - cmake_path(SET Python_Blosc2_INCLUDE NORMALIZE "${Python_INCLUDE}/blosc2.h") - cmake_path(HAS_FILENAME Python_Blosc2_INCLUDE HAS_BLOSC2) - if(HAS_BLOSC2) - set(BLOSC2_INCLUDE_DIR ${Python_INCLUDE}) - message(STATUS "Found Blosc2 include: ${Python_Blosc2_INCLUDE}") - else() - message(FATAL_ERROR "No Blosc2 includes found. Aborting.") - endif() +find_package(Python COMPONENTS Interpreter NumPy Development.Module REQUIRED) +message(STATUS "Executable: ${Python_EXECUTABLE}") +message(STATUS "NumPy found: ${Python_NumPy_FOUND}") +message(STATUS "NumPy version: ${Python_NumPy_VERSION}") +message(STATUS "Python_NumPy_INCLUDE_DIRS: ${Python_NumPy_INCLUDE_DIRS}") +if (UNIX) + cmake_path(SET Python_ROOT NORMALIZE "${Python_NumPy_INCLUDE_DIRS}/../../../../../..") else() - message(FATAL_ERROR "No Python found. Aborting.") + cmake_path(SET Python_ROOT NORMALIZE "${Python_NumPy_INCLUDE_DIRS}/../../../../..") +endif() +cmake_path(SET Python_INCLUDE NORMALIZE "${Python_ROOT}/include") +message(STATUS "Found Python include: ${Python_INCLUDE}") +cmake_path(SET Python_Blosc2_INCLUDE NORMALIZE "${Python_ROOT}/include/blosc2.h") +cmake_path(HAS_FILENAME Python_Blosc2_INCLUDE HAS_BLOSC2) +if(HAS_BLOSC2) + set(BLOSC2_INCLUDE_DIR ${Python_INCLUDE}) + message(STATUS "Found Blosc2 include: ${Python_Blosc2_INCLUDE}") +else() + message(FATAL_ERROR "No Blosc2 includes found. Aborting.") endif() include_directories("${BLOSC2_INCLUDE_DIR}") add_subdirectory(src) -add_subdirectory(examples) diff --git a/README.md b/README.md index 381dcb5..781bc5f 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,22 @@ This is an example on how to create a wheel of a plugin for Blosc2. ## Create the wheel +### For Linux + +```shell +python -m cibuildwheel --only 'cp311-manylinux_x86_64' +``` + +### For Mac + ```shell -python setup.py bdist_wheel +python -m cibuildwheel --only 'cp311-macosx_x86_64' +``` + +### For Windows + +```shell +python -m cibuildwheel --only 'cp311-win_amd64' ``` ## Verify the wheel is working @@ -33,34 +47,16 @@ drwxr-xr-x 3 faltet staff 96B Mar 6 13:45 __pycache__/ To test that the wheel has been installed and works properly. ```shell -cd _skbuild/*/cmake-build/examples/ -./test_plugin -Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$) -Compression ratio: 381.5 MB -> 2.9 MB (130.8x) -Compression time: 0.265 s, 1437.0 MB/s -Decompression time: 0.152 s, 2506.6 MB/s -Successful roundtrip data <-> schunk ! +cd examples/ +python schunk_roundtrip.py +Successful roundtrip! ``` If you want to see that the plugin example filter its being applied instead of any other use `BLOSC_TRACE=1`. -```shell -BLOSC_TRACE=1 ./test_plugin -Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$) -[info] - libpath for plugin blosc2_plugin_example: /Users/martaiborra/miniconda3/envs/plugin_example/lib/python3.11/site-packages/blosc2_plugin_example/libblosc2_plugin_example.so - (/Users/runner/work/python-blosc2/python-blosc2/blosc2/c-blosc2/blosc/blosc-private.h:265) -Inside plugin_example forward function - ... -Compression ratio: 381.5 MB -> 2.9 MB (130.8x) -Compression time: 0.25 s, 1523.0 MB/s -Inside plugin_example backward function - ... -Decompression time: 0.156 s, 2444.7 MB/s -Inside plugin_example backward function - ... -Successful roundtrip data <-> schunk ! -``` +There is also a C example that can be compiled and run following the instructions in its `test_plugin.c` +file. In the future, you should be able to test that the wheel is working with this command: diff --git a/blosc2_plugin_example/__init__.py b/blosc2_plugin_example/__init__.py index 625cc84..0aefe41 100644 --- a/blosc2_plugin_example/__init__.py +++ b/blosc2_plugin_example/__init__.py @@ -6,7 +6,6 @@ # # See LICENSE.txt for details about copyright and rights to use. -VERSION = 0.1 import os import platform @@ -18,7 +17,7 @@ def print_libpath(): if system in ["Linux", "Darwin"]: libname = "libblosc2_plugin_example.so" elif system == "Windows": - libname = "libblosc2_plugin_example.dll" + libname = "blosc2_plugin_example.dll" else: raise RuntimeError("Unsupported system: ", system) libpath = os.path.abspath(Path(__file__).parent / libname) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt deleted file mode 100644 index 10cb22c..0000000 --- a/examples/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -link_directories(${Python_LIB} ${Python_LIB64}) - -add_executable("test_plugin" "test_plugin.c") -target_link_libraries("test_plugin" blosc2) diff --git a/examples/array_roundtrip.py b/examples/array_roundtrip.py new file mode 100644 index 0000000..e554fb7 --- /dev/null +++ b/examples/array_roundtrip.py @@ -0,0 +1,27 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# This source code is licensed under a BSD-style license (found in the +# LICENSE file in the root directory of this source tree) +####################################################################### + + +import blosc2 +import numpy as np + + +# Register the plugin_example +blosc2.register_filter(250, None, None, "plugin_example") + +# Create blosc2 NDArray from a NumPy array +shape = [100, 100] +size = int(np.prod(shape)) +nparray = np.arange(size, dtype=np.int32).reshape(shape) +blosc2_array = blosc2.asarray(nparray, cparams={"filters": [250]}) + +# Retrieve data and check it is the same +if np.array_equal(blosc2_array[...], nparray): + print("Successful roundtrip!") +else: + print("Bad roundtrip! Try setting BLOSC_TRACE=1 envvar for more info on failure.") diff --git a/examples/schunk_roundtrip.py b/examples/schunk_roundtrip.py new file mode 100644 index 0000000..313017e --- /dev/null +++ b/examples/schunk_roundtrip.py @@ -0,0 +1,31 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# This source code is licensed under a BSD-style license (found in the +# LICENSE file in the root directory of this source tree) +####################################################################### + +import numpy as np + +import blosc2 + +blosc2.register_filter(250, None, None, "plugin_example") + +# Set the compression and decompression parameters +storage = {"contiguous": True, "cparams": {"filters": [250], "typesize": 4}} + +# Create the SChunk +nchunks = 10 +data = np.arange(200 * 1000 * nchunks, dtype=np.int32) +schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, data=data, **storage) + +cframe = schunk.to_cframe() + +schunk2 = blosc2.schunk_from_cframe(cframe, False) +data2 = np.empty(data.shape, dtype=data.dtype) +schunk2.get_slice(out=data2) +if np.array_equal(data, data2): + print("Successful roundtrip!") +else: + print("Bad roundtrip! Try setting BLOSC_TRACE=1 envvar for more info on failure.") diff --git a/examples/test_plugin.c b/examples/test_plugin.c index e8d47b7..7b9a946 100644 --- a/examples/test_plugin.c +++ b/examples/test_plugin.c @@ -7,6 +7,39 @@ Since it won't be registered yet as a filter/codec in Blosc2, it will be registered as user-defined. So it's id must be between 160-255. + To compile and run this test: + cd examples + gcc test_plugin.c -o test_plugin -I /Users/martaiborra/miniforge3/envs/plugin_example_arm64/include \ + -L /Users/martaiborra/miniforge3/envs/plugin_example_arm64/lib -lblosc2 + # For MacOS + DYLD_LIBRARY_PATH=/Users/martaiborra/miniforge3/envs/plugin_example_arm64/lib/ ./test_plugin + # For linux + LD_LIBRARY_PATH=path_to_blosc2_lib ./test_plugin + + To check that it is actually running the defined filter you can use BLOSC_TRACE=1. + BLOSC_TRACE=1 LD_LIBRARY_PATH=path_to_blosc2_lib ./test_plugin + + Expected output: + Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$) + Compression ratio: 381.5 MB -> 2.9 MB (130.8x) + Compression time: 0.265 s, 1437.0 MB/s + Decompression time: 0.152 s, 2506.6 MB/s + Successful roundtrip data <-> schunk ! + + Expected output with BLOSC_TRACE=1: + Blosc version info: 2.10.0 ($Date:: 2023-07-04 #$) + [info] - libpath for plugin blosc2_plugin_example: /Users/martaiborra/miniconda3/envs/plugin_example/lib/python3.11/site-packages/blosc2_plugin_example/libblosc2_plugin_example.so + (/Users/runner/work/python-blosc2/python-blosc2/blosc2/c-blosc2/blosc/blosc-private.h:265) + Inside plugin_example forward function + ... + Compression ratio: 381.5 MB -> 2.9 MB (130.8x) + Compression time: 0.25 s, 1523.0 MB/s + Inside plugin_example backward function + ... + Decompression time: 0.156 s, 2444.7 MB/s + Inside plugin_example backward function + ... + Successful roundtrip data <-> schunk ! */ diff --git a/pyproject.toml b/pyproject.toml index 5cbc879..79a97e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,12 +7,14 @@ # See LICENSE.txt for details about copyright and rights to use. [build-system] -requires = ["setuptools", "scikit-build", "cmake", "ninja", "blosc2"] -build-backend = "setuptools.build_meta" +#requires = ["scikit-build-core", "blosc2@git+https://github.com/Blosc/python-blosc2#egg=main"] +requires = ["scikit-build-core", "blosc2"] +build-backend = "scikit_build_core.build" [project] name = "blosc2_plugin_example" -dynamic = ["version", "readme"] +version = "0.0.1" +dynamic = ["readme"] authors = [ {name = "Blosc Development Team", email = "blosc@blosc.org"}, ] @@ -20,28 +22,34 @@ description = "My package description" keywords = ["plugin", "blosc2"] license = {text = "BSD-3-Clause"} classifiers = [ - "Framework :: Django", "Programming Language :: Python :: 3", - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Topic :: Software Development :: Libraries :: Python Modules", "Operating System :: Microsoft :: Windows", "Operating System :: Unix", "Programming Language :: C", - "Programming Language :: C++" ] dependencies = [ "blosc2" ] [tool.setuptools.dynamic] -version = {attr = "blosc2_plugin_example.VERSION"} readme = {file = ["README.md"], content-type = "text/markdown"} [tool.setuptools] platforms = [ "any" ] zip-safe = false +[tool.scikit-build] +# This activates verbose builds +cmake.verbose = true +# This controls the CMake build type +cmake.build-type = "Release" +# The licence file(s) to include in the wheel metadata directory. +wheel.license-files = ["LICEN[CS]E*", "COPYING*", "NOTICE*", "AUTHORS*"] + [tool.cibuildwheel] +#skip = "cp36-* cp37-* cp38-* cp39-* cp310-* pp37-* pp38-* pp39-* pp310-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" diff --git a/requirements-build.txt b/requirements-build.txt index 9c771bc..fed9b89 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1,3 +1,2 @@ -scikit-build -cmake +scikit-build-core blosc2 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ad30e73..0626586 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,5 +2,4 @@ add_library(blosc2_plugin_example MODULE urfilters.c) link_directories(${Python_LIB} ${Python_LIB64}) -# target_include_directories(blosc2_plugin_example PRIVATE ${BLOSC2_INCLUDE_DIR}) install(TARGETS blosc2_plugin_example LIBRARY DESTINATION blosc2_plugin_example) diff --git a/src/urfilters.c b/src/urfilters.c index ce896cf..23ba74c 100644 --- a/src/urfilters.c +++ b/src/urfilters.c @@ -10,21 +10,23 @@ #include "string.h" #include "blosc2.h" #include "blosc2/filters-registry.h" +#include "urfilters.h" #define FILTER_ID 250 #define FILTER_NAME "plugin_example" -#define FORWARD_NAME "blosc2_plugin_example_forward" -#define BACKWARD_NAME "blosc2_plugin_example_backward" - int blosc2_plugin_example_forward(const uint8_t* src, uint8_t* dest, int32_t size, uint8_t meta, blosc2_cparams *cparams, uint8_t id) { - char* blosc_trace = getenv("BLOSC_TRACE"); - if (blosc_trace != NULL) { - printf("Inside plugin_example forward function\n"); + BLOSC_TRACE_INFO("Inside plugin_example forward function\n"); + blosc2_schunk *schunk = NULL; + if (cparams != NULL && cparams->schunk != NULL) { + schunk = cparams->schunk; + } + else { + BLOSC_TRACE_ERROR("Cannot get schunk from cparams"); + return BLOSC2_ERROR_CODEC_PARAM; } - blosc2_schunk *schunk = cparams->schunk; for (int i = 0; i < size / schunk->typesize; ++i) { switch (schunk->typesize) { @@ -48,11 +50,15 @@ int blosc2_plugin_example_forward(const uint8_t* src, uint8_t* dest, int32_t siz int blosc2_plugin_example_backward(const uint8_t* src, uint8_t* dest, int32_t size, uint8_t meta, blosc2_dparams *dparams, uint8_t id) { - char* blosc_trace = getenv("BLOSC_TRACE"); - if (blosc_trace != NULL) { - printf("Inside plugin_example backward function\n"); + BLOSC_TRACE_INFO("Inside plugin_example backward function\n"); + blosc2_schunk *schunk = NULL; + if (dparams != NULL && dparams->schunk != NULL) { + schunk = dparams->schunk; + } + else { + BLOSC_TRACE_ERROR("Cannot get schunk from dparams"); + return BLOSC2_ERROR_CODEC_PARAM; } - blosc2_schunk *schunk = dparams->schunk; for (int i = 0; i < size / schunk->typesize; ++i) { switch (schunk->typesize) { @@ -74,4 +80,3 @@ int blosc2_plugin_example_backward(const uint8_t* src, uint8_t* dest, int32_t si } -filter_info info = {.forward=FORWARD_NAME, .backward=BACKWARD_NAME}; diff --git a/src/urfilters.h b/src/urfilters.h new file mode 100644 index 0000000..68f3946 --- /dev/null +++ b/src/urfilters.h @@ -0,0 +1,56 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 The Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + +**********************************************************************/ + +/********************************************************************* + @file urfilters.h + @brief Blosc2 plugin example header file. + + This file contains the two functions and defined filter info needed + to use the blosc2_plugin_example from Blosc2. + @author The Blosc Development Team +**********************************************************************/ + +#ifndef URFILTERS_H +#define URFILTERS_H + +#include +#include +#include +#include "blosc2.h" + + +// Needed for correctly exporting the functions and symbols for Windows +#if defined(_MSC_VER) +#define EXAMPLE_EXPORT __declspec(dllexport) +#elif (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__) +#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) + #define EXAMPLE_EXPORT __attribute__((dllexport)) +#else + #define EXAMPLE_EXPORT __attribute__((visibility("default"))) +#endif /* defined(_WIN32) || defined(__CYGWIN__) */ +#else +#error Cannot determine how to define EXAMPLE_EXPORT for this compiler. +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +EXAMPLE_EXPORT int blosc2_plugin_example_forward(const uint8_t* src, uint8_t* dest, int32_t size, uint8_t meta, + blosc2_cparams *cparams, uint8_t id); +EXAMPLE_EXPORT int blosc2_plugin_example_backward(const uint8_t* src, uint8_t* dest, int32_t size, uint8_t meta, + blosc2_dparams *dparams, uint8_t id); +EXAMPLE_EXPORT filter_info info = {"blosc2_plugin_example_forward", "blosc2_plugin_example_backward"}; + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_H */