diff --git a/src/c-blosc2/ANNOUNCE.md b/src/c-blosc2/ANNOUNCE.md index f6adb7d1..3947a853 100644 --- a/src/c-blosc2/ANNOUNCE.md +++ b/src/c-blosc2/ANNOUNCE.md @@ -1,12 +1,11 @@ -# Announcing C-Blosc2 2.9.2 +# Announcing C-Blosc2 2.10.2 A fast, compressed and persistent binary data store library for C. ## What is new? -This a maintenance release with improved support for dynamic plugins and -fixes for some corner cases when handling incompressible data. Also, -many other small fixes and improvements have been included. An upgrade to -this release is recommended. +This is a maintenance release with also several improvements for helping +integration of C-Blosc2 in other projects (thanks to Alex Huebl). Also, +some fixes for MinGW platform are in (thanks to Biswapriyo Nath). For more info, please see the release notes in: diff --git a/src/c-blosc2/Blosc2Config.cmake.in b/src/c-blosc2/Blosc2Config.cmake.in new file mode 100644 index 00000000..59274855 --- /dev/null +++ b/src/c-blosc2/Blosc2Config.cmake.in @@ -0,0 +1,84 @@ +# only add PUBLIC dependencies as well +# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-a-package-configuration-file +include(CMakeFindDependencyMacro) + +# Search in _ROOT: +# https://cmake.org/cmake/help/v3.12/policy/CMP0074.html +if(POLICY CMP0074) + cmake_policy(SET CMP0074 NEW) +endif() + +# locate the installed FindABC.cmake modules +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/Modules") + +# this section stores which configuration options were set +set(HAVE_THREADS @Threads_FOUND@) +set(HAVE_IPP @HAVE_IPP@) +set(HAVE_ZLIB_NG @HAVE_ZLIB_NG@) +set(DEACTIVATE_IPP @DEACTIVATE_IPP@) +set(DEACTIVATE_ZLIB @DEACTIVATE_ZLIB@) +set(DEACTIVATE_ZSTD @DEACTIVATE_ZSTD@) +set(PREFER_EXTERNAL_LZ4 @PREFER_EXTERNAL_LZ4@) +set(PREFER_EXTERNAL_ZLIB @PREFER_EXTERNAL_ZLIB@) +set(PREFER_EXTERNAL_ZSTD @PREFER_EXTERNAL_ZSTD@) + +# find dependencies and their targets, which are used in our Blosc2Targets.cmake +# additionally, the Blosc2_..._FOUND variables are used to support +# find_package(Blosc2 ... COMPONENTS ... ...) +# this enables downstream projects to express the need for specific features. +if(WIN32) + if(HAVE_THREADS) + find_dependency(Threads) + set(Blosc2_THREADS_FOUND TRUE) + else() + set(Blosc2_THREADS_FOUND FALSE) + endif() +else() + find_dependency(Threads) + set(Blosc2_THREADS_FOUND TRUE) +endif() + +if(NOT DEACTIVATE_IPP AND HAVE_IPP) + find_dependency(IPP) + set(Blosc2_IPP_FOUND FALSE) +else() + set(Blosc2_IPP_FOUND TRUE) +endif() + +if(PREFER_EXTERNAL_LZ4) + find_dependency(LZ4) +endif() +set(Blosc2_LZ4_FOUND TRUE) + +if(DEACTIVATE_ZLIB) + set(Blosc2_ZLIB_FOUND FALSE) +elseif(NOT DEACTIVATE_ZLIB AND PREFER_EXTERNAL_ZLIB) + if(HAVE_ZLIB_NG) + find_dependency(ZLIB_NG) + else() + find_dependency(ZLIB) + endif() + set(Blosc2_ZLIB_FOUND TRUE) +endif() + +if(DEACTIVATE_ZSTD) + set(Blosc2_ZSTD_FOUND FALSE) +elseif(NOT PREFER_EXTERNAL_ZSTD AND PREFER_EXTERNAL_ZSTD) + find_dependency(ZSTD) + set(Blosc2_ZSTD_FOUND TRUE) +endif() + +# define central Blosc2::blosc2_shared/static targets +include("${CMAKE_CURRENT_LIST_DIR}/Blosc2Targets.cmake") + +# check if components are fulfilled and set Blosc2__FOUND vars +# Blosc2_FIND_COMPONENTS is a list set by find_package(... COMPONENTS ... ...) +# likewise Blosc2_FIND_REQUIRED_... per component specified +foreach(comp ${Blosc2_FIND_COMPONENTS}) + if(NOT Blosc2_${comp}_FOUND) + if(Blosc2_FIND_REQUIRED_${comp}) + set(Blosc2_FOUND FALSE) + endif() + endif() +endforeach() + diff --git a/src/c-blosc2/CMakeLists.txt b/src/c-blosc2/CMakeLists.txt index aa846f1e..4b4fc6a5 100644 --- a/src/c-blosc2/CMakeLists.txt +++ b/src/c-blosc2/CMakeLists.txt @@ -240,7 +240,7 @@ endif() # Propagate CMAKE_OSX_ARCHITECTURES env variable into CMAKE_SYSTEM_PROCESSOR if(DEFINED ENV{CMAKE_OSX_ARCHITECTURES}) - if($ENV{CMAKE_OSX_ARCHITECTURES} STREQUAL "arm64") + if("$ENV{CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR arm64) endif() endif() @@ -385,23 +385,6 @@ if(NOT DEFINED BLOSC_INSTALL) endif() endif() -# uninstall target -if(BLOSC_INSTALL) - include(GNUInstallDirs) - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/blosc2.pc.in" - "${CMAKE_CURRENT_BINARY_DIR}/blosc2.pc" - @ONLY) - install(FILES "${CMAKE_CURRENT_BINARY_DIR}/blosc2.pc" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" COMPONENT DEV) - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" - "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" - @ONLY) - add_custom_target(uninstall - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) -endif() - # include directories include_directories(include) if(BUILD_PLUGINS) @@ -440,6 +423,126 @@ if(BUILD_EXAMPLES) add_subdirectory(examples) endif() +# collecting SOURCES is now complete +if(BUILD_SHARED) + target_sources(blosc2_shared PRIVATE ${SOURCES}) +endif() +if(BUILD_STATIC) + target_sources(blosc2_static PRIVATE ${SOURCES}) +endif() +if(BUILD_TESTS) + target_sources(blosc_testing PRIVATE ${SOURCES}) +endif() + +# install targets +if(BLOSC_INSTALL) + include(GNUInstallDirs) + + # C++ files + install(FILES ${PROJECT_SOURCE_DIR}/include/blosc2.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT DEV) + install(FILES ${PROJECT_SOURCE_DIR}/include/b2nd.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT DEV) + install(FILES + ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-export.h + ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-common.h + ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-stdio.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV) + if(BUILD_PLUGINS) + install(FILES + ${PROJECT_SOURCE_DIR}/include/blosc2/filters-registry.h + ${PROJECT_SOURCE_DIR}/include/blosc2/codecs-registry.h + ${PROJECT_SOURCE_DIR}/include/blosc2/tuners-registry.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV) + endif() + + if(BUILD_SHARED) + install(TARGETS blosc2_shared + LIBRARY COMPONENT LIB + ARCHIVE COMPONENT DEV + RUNTIME COMPONENT LIB) + endif() + if(BUILD_STATIC) + install(TARGETS blosc2_static COMPONENT DEV) + endif() + + # config files + include(CMakePackageConfigHelpers) + + # we need a general location for Unix and Windows to install our + # Blosc2Config.cmake files to. This is defined in CMake: + # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-search-procedure + if(NOT Blosc2_INSTALL_CMAKEDIR) + if(CMAKE_INSTALL_CMAKEDIR) + set(Blosc2_INSTALL_CMAKEDIR "${CMAKE_INSTALL_CMAKEDIR}/Blosc2") + else() + if(WIN32 AND NOT MINGW) + set(Blosc2_INSTALL_CMAKEDIR "cmake") + else() + set(Blosc2_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/Blosc2") + endif() + endif() + endif() + + # CMake config file + # This stores our targets and find and populates the targets we depend on, + # including third party interface libraries that we added. + set(Blosc2_INSTALL_TARGET_NAMES) + if(BUILD_SHARED) + list(APPEND Blosc2_INSTALL_TARGET_NAMES blosc2_shared) + endif() + if(BUILD_STATIC) + list(APPEND Blosc2_INSTALL_TARGET_NAMES blosc2_static) + endif() + configure_file( + ${PROJECT_SOURCE_DIR}/Blosc2Config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/Blosc2Config.cmake + @ONLY + ) + install(TARGETS ${Blosc2_INSTALL_TARGET_NAMES} + EXPORT Blosc2Targets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) + install(EXPORT Blosc2Targets + FILE Blosc2Targets.cmake + NAMESPACE Blosc2:: + DESTINATION ${Blosc2_INSTALL_CMAKEDIR} + ) + write_basic_package_version_file("Blosc2ConfigVersion.cmake" + VERSION ${BLOSC2_VERSION_STRING} + COMPATIBILITY SameMajorVersion + ) + install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/Blosc2Config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/Blosc2ConfigVersion.cmake + DESTINATION ${Blosc2_INSTALL_CMAKEDIR} + ) + + # CMake Find*.cmake files used in Blosc2Config.cmake + install( + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ + DESTINATION ${Blosc2_INSTALL_CMAKEDIR}/Modules + ) + + # pkg-config .pc file + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/blosc2.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/blosc2.pc" + @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/blosc2.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" COMPONENT DEV) + + # uninstaller + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" + @ONLY) + add_custom_target(uninstall + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) +endif() + # packaging if(NOT BLOSC_IS_SUBPROJECT) diff --git a/src/c-blosc2/README.rst b/src/c-blosc2/README.rst index 27720a0e..64f24557 100644 --- a/src/c-blosc2/README.rst +++ b/src/c-blosc2/README.rst @@ -70,7 +70,7 @@ New features in C-Blosc2 * **More filters:** besides `shuffle` and `bitshuffle` already present in C-Blosc1, C-Blosc2 already implements: - - `bytedelta`: calculates the difference between bytes in a block that has been shuffle already. We have `blogged about bytedelta `_. + - `bytedelta`: calculates the difference between bytes in a block that has been shuffled already. We have `blogged about bytedelta `_. - `delta`: the stored blocks inside a chunk are diff'ed with respect to first block in the chunk. The idea is that, in some situations, the diff will have more zeros than the original data, leading to better compression. @@ -227,6 +227,21 @@ Tweeter feed Follow `@Blosc2 `_ so as to get informed about the latest developments. +Citing Blosc +============ + +You can cite our work on the different libraries under the Blosc umbrella as: + +.. code-block:: console + + @ONLINE{blosc, + author = {{Blosc Development Team}}, + title = "{A fast, compressed and persistent data store library}", + year = {2009-2023}, + note = {https://blosc.org} + } + + Acknowledgments =============== diff --git a/src/c-blosc2/README_CHUNK_FORMAT.rst b/src/c-blosc2/README_CHUNK_FORMAT.rst index 2ac3a100..065fe534 100644 --- a/src/c-blosc2/README_CHUNK_FORMAT.rst +++ b/src/c-blosc2/README_CHUNK_FORMAT.rst @@ -9,8 +9,8 @@ A regular chunk is composed of a header and a blocks section:: Also, there are the so-called lazy chunks that do not have the actual compressed data, but only metainformation about how to read it. Lazy chunks typically appear when reading -data from persistent media. A lazy chunk has the header and bstarts sections in place -and in addition, they have an additional trailer for allowing to read the data blocks:: +data from persistent media. A lazy chunk has header and bstarts sections in place and +in addition, an additional trailer for allowing to read the data blocks:: +---------+---------+---------+ | header | bstarts | trailer | @@ -184,7 +184,7 @@ compression, and finally a list of compressed data streams:: | bstarts | dict | streams | +=========+======+=========+ -Each block is equal-sized as specified by the `blocksize` header field. The size of the last block that can be shorter +Each block is equal-sized as specified by the `blocksize` header field. The size of the last block can be shorter or equal to the rest. **Block starts** diff --git a/src/c-blosc2/RELEASE_NOTES.md b/src/c-blosc2/RELEASE_NOTES.md index 402e92de..dc6754ad 100644 --- a/src/c-blosc2/RELEASE_NOTES.md +++ b/src/c-blosc2/RELEASE_NOTES.md @@ -1,6 +1,63 @@ Release notes for C-Blosc2 ========================== +Changes from 2.10.1 to 2.10.2 +============================= + +* Several fixes for the CMake system. Thanks to Axel Huebl. See PR #541 and #542. + +* Several fixes for mingw plaform. Thanks to Biswapriyo Nath. See PR #540 and #543. + + +Changes from 2.10.0 to 2.10.1 +============================= + +* `blosc2_remove_urlpath(const char *urlpath)` does not return an error + when path does not exist. + +* Changes in CMake installer to conserve targets and properties + on install, so CMake users do not need to write `FindBlosc2.cmake` + files anymore. This also helps to preserve transitive dependencies on + CMake targets, especially useful for fully static builds, e.g., for + Python wheels. Thanks to @ax3l (Axel Huebl). See PR #537. + +* Fix new typos. Thanks to @DimitriPapadopoulos. See PR #538. + + +Changes from 2.9.3 to 2.10.0 +============================ + +* bytedelta filter has been fixed. For backward compatibility, the old + bytedelta filter is still available as `BLOSC_FILTER_BYTEDELTA_BUGGY` + symbol, with the same ID (34) than before. The new, fixed bytedelta + filter has received a new ID (35) and it can be used via the usual + `BLOSC_FILTER_BYTEDELTA` symbol. That means that old data written with + the buggy bytedelta filter should be decompressed without issues. + Thanks to @foody (Tom Birch) for the fix. See #531, #532 for more info. + +* Filter buffers are correctly cycled now. Now it is possible to use e.g. + shuffle and bitshuffle filters in the pipeline. Thanks to @foody (Tom Birch) + for the fix. See #528 and PR #530. + +* Assorted fixes for allowing better inclusion in external projects. + Thanks to @ax3l (Axel Huebel). See #525, #527 and #529. + +* Minor fixes in the documentation. Thanks to @ivilata (Ivan Vilata). + See #523. + + +Changes from 2.9.2 to 2.9.3 +=========================== + +* Thanks to Dimitri Papadopoulos for an extensive set of improvements in + documentation and code. + +* `load_lib` is now a private function. Before was public, but + never meant to be. + +* Several fixes for bugs discovered by the fuzzer. + + Changes from 2.9.1 to 2.9.2 =========================== diff --git a/src/c-blosc2/RELEASING.rst b/src/c-blosc2/RELEASING.rst index 1d39c61b..5ddc647d 100644 --- a/src/c-blosc2/RELEASING.rst +++ b/src/c-blosc2/RELEASING.rst @@ -93,8 +93,7 @@ Post-release actions version to the next minor one (i.e. X.Y.Z --> X.Y.(Z+1).dev). - Create new headers for adding new features in ``RELEASE_NOTES.md`` - and empty the release-specific information in ``ANNOUNCE.md`` and - add this place-holder instead: + and add this place-holder instead: #XXX version-specific blurb XXX# diff --git a/src/c-blosc2/THANKS.rst b/src/c-blosc2/THANKS.rst index 75323563..2b8f8e3c 100644 --- a/src/c-blosc2/THANKS.rst +++ b/src/c-blosc2/THANKS.rst @@ -30,4 +30,6 @@ Thanks * Nathan Moinvaziri for his outstanding work on the security side of the things via `fuzzer testing `_. -* Marta Iborra for her implementation of sparse storage for persistent super-chunks. +* Marta Iborra for her implementation of sparse storage for persistent super-chunks and her attention to detail in many other aspects of the library. + +* Dimitri Papadopoulos for an extensive set of improvements in documentation and code. diff --git a/src/c-blosc2/bench/b2nd/CMakeLists.txt b/src/c-blosc2/bench/b2nd/CMakeLists.txt index 65c45e31..89a4f6f6 100644 --- a/src/c-blosc2/bench/b2nd/CMakeLists.txt +++ b/src/c-blosc2/bench/b2nd/CMakeLists.txt @@ -12,5 +12,5 @@ foreach (source ${SOURCES}) get_filename_component(target_name ${source} NAME_WE) set(target b2nd_${target_name}) add_executable(${target} ${target_name}.c) - target_link_libraries(${target} blosc_testing ${LIBS}) + target_link_libraries(${target} PUBLIC blosc_testing ${LIBS}) endforeach (source) diff --git a/src/c-blosc2/bench/b2nd/bench_zfp_getitem.c b/src/c-blosc2/bench/b2nd/bench_zfp_getitem.c index 6e91cb41..079f14fd 100644 --- a/src/c-blosc2/bench/b2nd/bench_zfp_getitem.c +++ b/src/c-blosc2/bench/b2nd/bench_zfp_getitem.c @@ -43,11 +43,12 @@ precip0[:] = values * */ -#include -#include "../../include/blosc2/codecs-registry.h" +#include "../plugins/codecs/zfp/zfp-private.h" #include "../../plugins/codecs/zfp/blosc2-zfp.h" -#include -#include +#include "context.h" +#include "blosc2/codecs-registry.h" +#include "b2nd.h" +#include "blosc2.h" int comp(const char *urlpath) { blosc2_init(); diff --git a/src/c-blosc2/bench/sframe_bench.c b/src/c-blosc2/bench/sframe_bench.c index c0f2dfd8..2c76275a 100644 --- a/src/c-blosc2/bench/sframe_bench.c +++ b/src/c-blosc2/bench/sframe_bench.c @@ -165,9 +165,9 @@ void test_insert(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) { cframe_insert_time += blosc_elapsed_secs(current, last); } - printf("[Sframe Insert] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes \n", + printf("[Sframe Insert] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes\n", sframe_insert_time, schunk_sframe->cbytes); - printf("[Cframe Insert] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes \n", + printf("[Cframe Insert] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes\n", cframe_insert_time, schunk_cframe->cbytes); @@ -208,9 +208,9 @@ void test_reorder(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) { } cframe_reorder_time = blosc_elapsed_secs(current, last); - printf("[Sframe Update] Elapsed time:\t %f s. Total sframe size: %.3" PRId64 " bytes \n", + printf("[Sframe Update] Elapsed time:\t %f s. Total sframe size: %.3" PRId64 " bytes\n", sframe_reorder_time, schunk_sframe->cbytes); - printf("[Cframe Update] Elapsed time:\t %f s. Total cframe size: %.3" PRId64 " bytes \n", + printf("[Cframe Update] Elapsed time:\t %f s. Total cframe size: %.3" PRId64 " bytes\n", cframe_reorder_time, schunk_cframe->cbytes); /* Free resources */ diff --git a/src/c-blosc2/blosc/CMakeLists.txt b/src/c-blosc2/blosc/CMakeLists.txt index 86f6668f..b44b7107 100644 --- a/src/c-blosc2/blosc/CMakeLists.txt +++ b/src/c-blosc2/blosc/CMakeLists.txt @@ -9,71 +9,161 @@ # A simple way to detect that we are using CMAKE add_definitions(-DUSING_CMAKE) -set(INTERNAL_LIBS ${PROJECT_SOURCE_DIR}/internal-complibs) +set(version_string ${BLOSC2_VERSION_MAJOR}.${BLOSC2_VERSION_MINOR}.${BLOSC2_VERSION_PATCH}) -# Hide symbols by default unless they're specifically exported. -# This makes it easier to keep the set of exported symbols the -# same across all compilers/platforms. -set(CMAKE_C_VISIBILITY_PRESET hidden) +# targets +if(BUILD_SHARED) + add_library(blosc2_shared SHARED) + # ALIAS for superbuilds that use Blosc2 as sub-project + # must be the same as the NAMESPACE in Blosc2Targets + add_library(Blosc2::blosc2_shared ALIAS blosc2_shared) + set_target_properties(blosc2_shared PROPERTIES + OUTPUT_NAME blosc2 + # Hide symbols by default unless they're specifically exported. + # This makes it easier to keep the set of exported symbols the + # same across all compilers/platforms. + C_VISIBILITY_PRESET hidden + ) + if(MSVC OR MINGW) + set_target_properties(blosc2_shared PROPERTIES PREFIX lib) + endif() + set_target_properties(blosc2_shared PROPERTIES + VERSION ${version_string} + SOVERSION 2 # Change this when an ABI change happens + ) + target_compile_definitions(blosc2_shared PUBLIC BLOSC_SHARED_LIBRARY) + target_include_directories(blosc2_shared PUBLIC + $ + $) +endif() +if(BUILD_STATIC) + add_library(blosc2_static STATIC) + # ALIAS for superbuilds that use Blosc2 as sub-project + # must be the same as the NAMESPACE in Blosc2Targets + add_library(Blosc2::blosc2_static ALIAS blosc2_static) + set_target_properties(blosc2_static PROPERTIES + OUTPUT_NAME blosc2 + POSITION_INDEPENDENT_CODE ON + # Hide symbols by default unless they're specifically exported. + # This makes it easier to keep the set of exported symbols the + # same across all compilers/platforms. + C_VISIBILITY_PRESET hidden + ) + if(MSVC OR MINGW) + set_target_properties(blosc2_static PROPERTIES PREFIX lib) + endif() + target_include_directories(blosc2_static PUBLIC + $ + $) +endif() +# When the option has been selected to compile the test suite, +# compile an additional version of blosc2_static which exports +# some normally-hidden symbols (to facilitate unit testing). +if(BUILD_TESTS) + add_library(blosc_testing STATIC) + set_target_properties(blosc_testing PROPERTIES OUTPUT_NAME blosc_testing) + if(MSVC OR MINGW) + set_target_properties(blosc_testing PROPERTIES PREFIX lib) + endif() + target_compile_definitions(blosc_testing PUBLIC + BLOSC_TESTING + BLOSC_SHARED_LIBRARY # for EXPORT macro + ) + target_include_directories(blosc_testing PUBLIC + $ + $) +endif() + +set(INTERNAL_LIBS ${PROJECT_SOURCE_DIR}/internal-complibs) -# includes -set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}) +# link dependencies +# "link" dependent targets via target_link_libraries (preferred) and +# manually add includes / libs for others if(LZ4_FOUND) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_INCLUDE_DIR}) + if(BUILD_SHARED) + target_include_directories(blosc2_shared PUBLIC ${LZ4_INCLUDE_DIR}) + endif() + if(BUILD_STATIC) + target_include_directories(blosc2_static PUBLIC ${LZ4_INCLUDE_DIR}) + endif() + if(BUILD_TESTS) + target_include_directories(blosc_testing PUBLIC ${LZ4_INCLUDE_DIR}) + endif() else() set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.9.4) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_LOCAL_DIR}) + if(BUILD_SHARED) + target_include_directories(blosc2_shared PRIVATE ${LZ4_LOCAL_DIR}) + endif() + if(BUILD_STATIC) + target_include_directories(blosc2_static PRIVATE ${LZ4_LOCAL_DIR}) + endif() + if(BUILD_TESTS) + target_include_directories(blosc_testing PRIVATE ${LZ4_LOCAL_DIR}) + endif() endif() if(NOT DEACTIVATE_ZLIB) if(ZLIB_NG_FOUND) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_NG_INCLUDE_DIR}) + if(BUILD_SHARED) + target_link_libraries(blosc2_shared PUBLIC ZLIB_NG::ZLIB_NG) + endif() + if(BUILD_STATIC) + target_link_libraries(blosc2_static PUBLIC ZLIB_NG::ZLIB_NG) + endif() + if(BUILD_TESTS) + target_link_libraries(blosc_testing PUBLIC ZLIB_NG::ZLIB_NG) + endif() elseif(ZLIB_FOUND) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR}) + if(BUILD_SHARED) + target_link_libraries(blosc2_shared PUBLIC ZLIB::ZLIB) + endif() + if(BUILD_STATIC) + target_link_libraries(blosc2_static PUBLIC ZLIB::ZLIB) + endif() + if(BUILD_TESTS) + target_link_libraries(blosc_testing PUBLIC ZLIB::ZLIB) + endif() else() set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/${ZLIB_NG_DIR}) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_LOCAL_DIR}) + if(BUILD_SHARED) + target_include_directories(blosc2_shared PRIVATE ${ZLIB_LOCAL_DIR}) + endif() + if(BUILD_STATIC) + target_include_directories(blosc2_static PRIVATE ${ZLIB_LOCAL_DIR}) + endif() + if(BUILD_TESTS) + target_include_directories(blosc_testing PRIVATE ${ZLIB_LOCAL_DIR}) + endif() endif() endif() if(NOT DEACTIVATE_ZSTD) if(ZSTD_FOUND) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIR}) + if(BUILD_SHARED) + target_include_directories(blosc2_shared PUBLIC ${ZSTD_INCLUDE_DIR}) + target_link_libraries(blosc2_shared PUBLIC ${ZSTD_LIBRARY}) + endif() + if(BUILD_STATIC) + target_include_directories(blosc2_static PUBLIC ${ZSTD_INCLUDE_DIR}) + target_link_libraries(blosc2_static PUBLIC ${ZSTD_LIBRARY}) + endif() + if(BUILD_TESTS) + target_include_directories(blosc_testing PUBLIC ${ZSTD_INCLUDE_DIR}) + target_link_libraries(blosc_testing PUBLIC ${ZSTD_LIBRARY}) + endif() else() set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.5.5) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_LOCAL_DIR} - ${ZSTD_LOCAL_DIR}/common) - endif() -endif() - -include_directories(${BLOSC_INCLUDE_DIRS}) - -# library sources -set(SOURCES ${SOURCES} blosc2.c blosclz.c fastcopy.c fastcopy.h schunk.c frame.c stune.c stune.h - context.h delta.c delta.h shuffle-generic.c bitshuffle-generic.c trunc-prec.c trunc-prec.h - timestamp.c sframe.c directories.c blosc2-stdio.c - b2nd.c b2nd_utils.c) -if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL arm64) - if(COMPILER_SUPPORT_SSE2) - message(STATUS "Adding run-time support for SSE2") - set(SOURCES ${SOURCES} shuffle-sse2.c bitshuffle-sse2.c) - endif() - if(COMPILER_SUPPORT_AVX2) - message(STATUS "Adding run-time support for AVX2") - set(SOURCES ${SOURCES} shuffle-avx2.c bitshuffle-avx2.c) + if(BUILD_SHARED) + target_include_directories(blosc2_shared PRIVATE ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common) + endif() + if(BUILD_STATIC) + target_include_directories(blosc2_static PRIVATE ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common) + endif() + if(BUILD_TESTS) + target_include_directories(blosc_testing PRIVATE ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common) + endif() endif() endif() -if(COMPILER_SUPPORT_NEON) - message(STATUS "Adding run-time support for NEON") - set(SOURCES ${SOURCES} shuffle-neon.c bitshuffle-neon.c) -endif() -if(COMPILER_SUPPORT_ALTIVEC) - message(STATUS "Adding run-time support for ALTIVEC") - set(SOURCES ${SOURCES} shuffle-altivec.c bitshuffle-altivec.c) -endif() -set(SOURCES ${SOURCES} shuffle.c) - -set(version_string ${BLOSC2_VERSION_MAJOR}.${BLOSC2_VERSION_MINOR}.${BLOSC2_VERSION_PATCH}) set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1 set(THREADS_PREFER_PTHREAD_FLAG TRUE) # CMake 3.1+ @@ -82,7 +172,7 @@ if(WIN32) find_package(Threads) if(NOT Threads_FOUND) message(STATUS "using the internal pthread library for win32 systems.") - set(SOURCES ${SOURCES} win32/pthread.c) + list(APPEND SOURCES blosc/win32/pthread.c) else() if(CMAKE_VERSION VERSION_LESS 3.1) set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) @@ -97,13 +187,14 @@ else() else() set(LIBS ${LIBS} Threads::Threads) endif() + set(LIBS ${LIBS} ${CMAKE_DL_LIBS}) endif() if(LZ4_FOUND) set(LIBS ${LIBS} ${LZ4_LIBRARY}) else() file(GLOB LZ4_FILES ${LZ4_LOCAL_DIR}/*.c) - set(SOURCES ${SOURCES} ${LZ4_FILES}) + list(APPEND SOURCES ${LZ4_FILES}) source_group("LZ4" FILES ${LZ4_FILES}) endif() @@ -111,11 +202,11 @@ if(NOT DEACTIVATE_ZLIB) if(ZLIB_NG_FOUND) set(LIBS ${LIBS} ${ZLIB_NG_LIBRARY}) elseif(ZLIB_FOUND) - set(LIBS ${LIBS} ${ZLIB_LIBRARY}) + set(LIBS ${LIBS} ${ZLIB_LIBRARIES}) else() set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/${ZLIB_NG_DIR}) file(GLOB ZLIB_FILES ${ZLIB_LOCAL_DIR}/*.c) - set(SOURCES ${SOURCES} ${ZLIB_FILES}) + list(APPEND SOURCES ${ZLIB_FILES}) source_group("Zlib" FILES ${ZLIB_FILES}) endif() endif() @@ -139,7 +230,7 @@ if(NOT DEACTIVATE_ZSTD) file(GLOB ZSTD_DICT_FILES ${ZSTD_LOCAL_DIR}/dictBuilder/*.c) set(ZSTD_FILES ${ZSTD_COMMON_FILES} ${ZSTD_COMPRESS_FILES} ${ZSTD_DECOMPRESS_FILES} ${ZSTD_DICT_FILES}) - set(SOURCES ${SOURCES} ${ZSTD_FILES}) + list(APPEND SOURCES ${ZSTD_FILES}) source_group("Zstd" FILES ${ZSTD_FILES}) endif() endif() @@ -155,21 +246,49 @@ if(UNIX AND NOT APPLE) endif() -# targets -if(BUILD_SHARED) - add_library(blosc2_shared SHARED ${SOURCES}) - set_target_properties(blosc2_shared PROPERTIES OUTPUT_NAME blosc2) - if(MSVC OR MINGW) - set_target_properties(blosc2_shared PROPERTIES PREFIX lib) +# Blosc2 library source files +list(APPEND SOURCES + blosc/blosc2.c + blosc/blosclz.c + blosc/fastcopy.c + blosc/fastcopy.h + blosc/schunk.c + blosc/frame.c + blosc/stune.c + blosc/stune.h + blosc/context.h + blosc/delta.c + blosc/delta.h + blosc/shuffle-generic.c + blosc/bitshuffle-generic.c + blosc/trunc-prec.c + blosc/trunc-prec.h + blosc/timestamp.c + blosc/sframe.c + blosc/directories.c + blosc/blosc2-stdio.c + blosc/b2nd.c + blosc/b2nd_utils.c +) +if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL arm64) + if(COMPILER_SUPPORT_SSE2) + message(STATUS "Adding run-time support for SSE2") + list(APPEND SOURCES blosc/shuffle-sse2.c blosc/bitshuffle-sse2.c) endif() - set_target_properties(blosc2_shared PROPERTIES - VERSION ${version_string} - SOVERSION 2 # Change this when an ABI change happens - ) - set_property( - TARGET blosc2_shared - APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY) + if(COMPILER_SUPPORT_AVX2) + message(STATUS "Adding run-time support for AVX2") + list(APPEND SOURCES blosc/shuffle-avx2.c blosc/bitshuffle-avx2.c) + endif() +endif() +if(COMPILER_SUPPORT_NEON) + message(STATUS "Adding run-time support for NEON") + list(APPEND SOURCES blosc/shuffle-neon.c blosc/bitshuffle-neon.c) +endif() +if(COMPILER_SUPPORT_ALTIVEC) + message(STATUS "Adding run-time support for ALTIVEC") + list(APPEND SOURCES blosc/shuffle-altivec.c blosc/bitshuffle-altivec.c) endif() +list(APPEND SOURCES blosc/shuffle.c) # Based on the target architecture and hardware features supported # by the C compiler, set hardware architecture optimization flags @@ -180,7 +299,7 @@ if(COMPILER_SUPPORT_SSE2) if(${CMAKE_SIZEOF_VOID_P} EQUAL 4) set_source_files_properties( shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c - PROPERTIES COMPILE_FLAGS "/arch:SSE2") + PROPERTIES COMPILE_OPTIONS "/arch:SSE2") set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_OPTIONS "/arch:SSE2") @@ -188,7 +307,7 @@ if(COMPILER_SUPPORT_SSE2) else() set_source_files_properties( shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c - PROPERTIES COMPILE_FLAGS -msse2) + PROPERTIES COMPILE_OPTIONS -msse2) set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_OPTIONS -msse2) @@ -210,14 +329,14 @@ if(COMPILER_SUPPORT_AVX2) if(MSVC) set_source_files_properties( shuffle-avx2.c bitshuffle-avx2.c - PROPERTIES COMPILE_FLAGS "/arch:AVX2") + PROPERTIES COMPILE_OPTIONS "/arch:AVX2") set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_OPTIONS "/arch:AVX2") else() set_source_files_properties( shuffle-avx2.c bitshuffle-avx2.c - PROPERTIES COMPILE_FLAGS -mavx2) + PROPERTIES COMPILE_OPTIONS -mavx2) set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_OPTIONS -mavx2) @@ -233,7 +352,7 @@ endif() if(COMPILER_SUPPORT_NEON) set_source_files_properties( shuffle-neon.c bitshuffle-neon.c - PROPERTIES COMPILE_FLAGS "-flax-vector-conversions") + PROPERTIES COMPILE_OPTIONS "-flax-vector-conversions") set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_OPTIONS "-flax-vector-conversions") @@ -241,7 +360,7 @@ if(COMPILER_SUPPORT_NEON) # Only armv7l needs special -mfpu=neon flag; aarch64 doesn't. set_source_files_properties( shuffle-neon.c bitshuffle-neon.c - PROPERTIES COMPILE_FLAGS "-mfpu=neon -flax-vector-conversions") + PROPERTIES COMPILE_OPTIONS "-mfpu=neon -flax-vector-conversions") set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_OPTIONS "-mfpu=neon -flax-vector-conversions") @@ -256,7 +375,7 @@ endif() if(COMPILER_SUPPORT_ALTIVEC) set_source_files_properties( shuffle-altivec.c bitshuffle-altivec.c - PROPERTIES COMPILE_FLAGS -DNO_WARN_X86_INTRINSICS) + PROPERTIES COMPILE_OPTIONS -DNO_WARN_X86_INTRINSICS) set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_OPTIONS -DNO_WARN_X86_INTRINSICS) @@ -269,69 +388,21 @@ if(COMPILER_SUPPORT_ALTIVEC) APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_ALTIVEC_ENABLED) endif() -# When the option has been selected to compile the test suite, -# compile an additional version of blosc2_static which exports -# some normally-hidden symbols (to facilitate unit testing). -if(BUILD_TESTS) - add_library(blosc_testing STATIC ${SOURCES}) - set_target_properties(blosc_testing PROPERTIES OUTPUT_NAME blosc_testing) - if(MSVC OR MINGW) - set_target_properties(blosc_testing PROPERTIES PREFIX lib) - endif() - set_property( - TARGET blosc_testing - APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY) - set_property( - TARGET blosc_testing - APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) -endif() - +# add libraries for dependencies that are not CMake targets if(BUILD_SHARED) - target_link_libraries(blosc2_shared ${LIBS}) + target_link_libraries(blosc2_shared PUBLIC ${LIBS}) target_include_directories(blosc2_shared PUBLIC ${BLOSC_INCLUDE_DIRS}) endif() -if(BUILD_TESTS) - target_link_libraries(blosc_testing ${LIBS}) - target_include_directories(blosc_testing PUBLIC ${BLOSC_INCLUDE_DIRS}) -endif() - if(BUILD_STATIC) - add_library(blosc2_static STATIC ${SOURCES}) - set_target_properties(blosc2_static PROPERTIES OUTPUT_NAME blosc2) - set_target_properties(blosc2_static PROPERTIES POSITION_INDEPENDENT_CODE ON) - if(MSVC OR MINGW) - set_target_properties(blosc2_static PROPERTIES PREFIX lib) - endif() - target_link_libraries(blosc2_static ${LIBS}) + target_link_libraries(blosc2_static PUBLIC ${LIBS}) target_include_directories(blosc2_static PUBLIC ${BLOSC_INCLUDE_DIRS}) endif() -# install -if(BLOSC_INSTALL) - install(FILES ${PROJECT_SOURCE_DIR}/include/blosc2.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT DEV) - install(FILES ${PROJECT_SOURCE_DIR}/include/b2nd.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT DEV) - install(FILES - ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-export.h - ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-common.h - ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-stdio.h - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV) - if(BUILD_PLUGINS) - install(FILES - ${PROJECT_SOURCE_DIR}/include/blosc2/filters-registry.h - ${PROJECT_SOURCE_DIR}/include/blosc2/codecs-registry.h - ${PROJECT_SOURCE_DIR}/include/blosc2/plugins-utils.h - ${PROJECT_SOURCE_DIR}/include/blosc2/tuners-registry.h - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV) - endif() - - if(BUILD_SHARED) - install(TARGETS blosc2_shared - LIBRARY COMPONENT LIB - ARCHIVE COMPONENT DEV - RUNTIME COMPONENT LIB) - endif() - if(BUILD_STATIC) - install(TARGETS blosc2_static COMPONENT DEV) - endif() +if(BUILD_TESTS) + target_link_libraries(blosc_testing PUBLIC ${LIBS}) + target_include_directories(blosc_testing PUBLIC ${BLOSC_INCLUDE_DIRS}) endif() + +# we use this variable in the CMake file one directory above ours +set(SOURCES ${SOURCES} PARENT_SCOPE) diff --git a/src/c-blosc2/blosc/b2nd.c b/src/c-blosc2/blosc/b2nd.c index f2cf77ad..4c0ff70c 100644 --- a/src/c-blosc2/blosc/b2nd.c +++ b/src/c-blosc2/blosc/b2nd.c @@ -9,12 +9,15 @@ **********************************************************************/ #include "b2nd.h" -#include "context.h" #include "b2nd_utils.h" -#include "blosc2.h" +#include "context.h" #include "blosc2/blosc2-common.h" +#include "blosc2.h" #include +#include +#include +#include int b2nd_serialize_meta(int8_t ndim, const int64_t *shape, const int32_t *chunkshape, @@ -266,6 +269,10 @@ int array_new(b2nd_context_t *ctx, int special_value, b2nd_array_t **array) { } } + if ((*array)->extchunknitems * sc->typesize > BLOSC2_MAX_BUFFERSIZE){ + BLOSC_TRACE_ERROR("Chunksize exceeds maximum of %d", BLOSC2_MAX_BUFFERSIZE); + return BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED; + } // Fill schunk with uninit values if ((*array)->nitems != 0) { int32_t chunksize = (int32_t) (*array)->extchunknitems * sc->typesize; @@ -1093,7 +1100,7 @@ int b2nd_print_meta(const b2nd_array_t *array) { &dtype, &dtype_format)); free(smeta); - printf("b2nd metalayer parameters: \n Ndim: %d", ndim); + printf("b2nd metalayer parameters:\n Ndim: %d", ndim); printf("\n shape: %" PRId64 "", shape[0]); for (int i = 1; i < ndim; ++i) { printf(", %" PRId64 "", shape[i]); diff --git a/src/c-blosc2/blosc/b2nd_utils.c b/src/c-blosc2/blosc/b2nd_utils.c index 4b4ff7d9..bcd5e141 100644 --- a/src/c-blosc2/blosc/b2nd_utils.c +++ b/src/c-blosc2/blosc/b2nd_utils.c @@ -9,6 +9,10 @@ **********************************************************************/ #include "b2nd_utils.h" +#include "b2nd.h" + +#include +#include // copyNdim where N = {2-8} - specializations of copy loops to be used by b2nd_copy_buffer // since we don't have c++ templates, substitute manual specializations for up to known B2ND_MAX_DIM (8) diff --git a/src/c-blosc2/blosc/b2nd_utils.h b/src/c-blosc2/blosc/b2nd_utils.h index 2ba021af..38fba67a 100644 --- a/src/c-blosc2/blosc/b2nd_utils.h +++ b/src/c-blosc2/blosc/b2nd_utils.h @@ -8,16 +8,10 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef B2ND_B2ND_UTILS_H_ -#define B2ND_B2ND_UTILS_H_ - -#include -#include <../plugins/plugin_utils.h> - -#ifdef __cplusplus -extern "C" { -#endif +#ifndef BLOSC_B2ND_UTILS_H +#define BLOSC_B2ND_UTILS_H +#include int b2nd_copy_buffer(int8_t ndim, uint8_t itemsize, @@ -26,8 +20,4 @@ int b2nd_copy_buffer(int8_t ndim, void *dst, const int64_t *dst_pad_shape, int64_t *dst_start); -#ifdef __cplusplus -} -#endif - -#endif // B2ND_B2ND_UTILS_H_ +#endif /* BLOSC_B2ND_UTILS_H */ diff --git a/src/c-blosc2/blosc/bitshuffle-altivec.c b/src/c-blosc2/blosc/bitshuffle-altivec.c index 7042e0ed..884bf90c 100644 --- a/src/c-blosc2/blosc/bitshuffle-altivec.c +++ b/src/c-blosc2/blosc/bitshuffle-altivec.c @@ -28,9 +28,12 @@ /* Make sure ALTIVEC is available for the compilation target and compiler. */ #if defined(__ALTIVEC__) -#include #include "transpose-altivec.h" +#include + +#include + /* The next is useful for debugging purposes */ #if 0 #include diff --git a/src/c-blosc2/blosc/bitshuffle-altivec.h b/src/c-blosc2/blosc/bitshuffle-altivec.h index 000fab21..5995aa74 100644 --- a/src/c-blosc2/blosc/bitshuffle-altivec.h +++ b/src/c-blosc2/blosc/bitshuffle-altivec.h @@ -10,15 +10,13 @@ /* ALTIVEC-accelerated shuffle/unshuffle routines. */ -#ifndef BITSHUFFLE_ALTIVEC_H -#define BITSHUFFLE_ALTIVEC_H +#ifndef BLOSC_BITSHUFFLE_ALTIVEC_H +#define BLOSC_BITSHUFFLE_ALTIVEC_H #include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif - +#include +#include BLOSC_NO_EXPORT int64_t bshuf_trans_byte_elem_altivec(void* in, void* out, const size_t size, @@ -46,9 +44,4 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_altivec(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); -#ifdef __cplusplus -} -#endif - - -#endif /* BITSHUFFLE_ALTIVEC_H */ +#endif /* BLOSC_BITSHUFFLE_ALTIVEC_H */ diff --git a/src/c-blosc2/blosc/bitshuffle-avx2.c b/src/c-blosc2/blosc/bitshuffle-avx2.c index c2014544..a855bf83 100644 --- a/src/c-blosc2/blosc/bitshuffle-avx2.c +++ b/src/c-blosc2/blosc/bitshuffle-avx2.c @@ -20,7 +20,6 @@ rights to use. **********************************************************************/ - #include "bitshuffle-avx2.h" #include "bitshuffle-sse2.h" #include "bitshuffle-generic.h" @@ -30,6 +29,8 @@ #include +#include + /* The next is useful for debugging purposes */ #if 0 #include diff --git a/src/c-blosc2/blosc/bitshuffle-avx2.h b/src/c-blosc2/blosc/bitshuffle-avx2.h index e68649e7..2b40bd29 100644 --- a/src/c-blosc2/blosc/bitshuffle-avx2.h +++ b/src/c-blosc2/blosc/bitshuffle-avx2.h @@ -10,14 +10,13 @@ /* AVX2-accelerated shuffle/unshuffle routines. */ -#ifndef BITSHUFFLE_AVX2_H -#define BITSHUFFLE_AVX2_H +#ifndef BLOSC_BITSHUFFLE_AVX2_H +#define BLOSC_BITSHUFFLE_AVX2_H -#include +#include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif +#include +#include /** AVX2-accelerated bitshuffle routine. @@ -33,8 +32,4 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_avx2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); -#ifdef __cplusplus -} -#endif - -#endif /* BITSHUFFLE_AVX2_H */ +#endif /* BLOSC_BITSHUFFLE_AVX2_H */ diff --git a/src/c-blosc2/blosc/bitshuffle-generic.c b/src/c-blosc2/blosc/bitshuffle-generic.c index ba62c66f..13748efd 100644 --- a/src/c-blosc2/blosc/bitshuffle-generic.c +++ b/src/c-blosc2/blosc/bitshuffle-generic.c @@ -10,6 +10,8 @@ #include "bitshuffle-generic.h" +#include + #ifdef _MSC_VER #pragma warning (push) #pragma warning (disable: 4146) diff --git a/src/c-blosc2/blosc/bitshuffle-generic.h b/src/c-blosc2/blosc/bitshuffle-generic.h index 88c8388d..843b66a5 100644 --- a/src/c-blosc2/blosc/bitshuffle-generic.h +++ b/src/c-blosc2/blosc/bitshuffle-generic.h @@ -8,28 +8,39 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +/* + * Bitshuffle - Filter for improving compression of typed binary data. + * + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + */ + /* Generic (non-hardware-accelerated) shuffle/unshuffle routines. These are used when hardware-accelerated functions aren't available for a particular platform; they are also used by the hardware- accelerated functions to handle any remaining elements in a block which isn't a multiple of the hardware's vector size. */ -#ifndef BITSHUFFLE_GENERIC_H -#define BITSHUFFLE_GENERIC_H +#ifndef BLOSC_BITSHUFFLE_GENERIC_H +#define BLOSC_BITSHUFFLE_GENERIC_H -#include -#include +#include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif +#include +#include +// Macros. +#define CHECK_MULT_EIGHT(n) if (n % 8) return -80; +#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) -/* Macros. */ -#define CHECK_MULT_EIGHT(n) if ((n) % 8) return -80; #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) -#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) -#define CHECK_ERR(count) if ((count) < 0) { return count; } +#define CHECK_ERR(count) \ + do { \ + if ((count) < 0) \ + return count; \ + } while (0) /* ---- Worker code not requiring special instruction sets. ---- @@ -42,41 +53,41 @@ extern "C" { /* Transpose 8x8 bit array packed into a single quadword *x*. * *t* is workspace. */ #define TRANS_BIT_8X8(x, t) { \ - (t) = ((x) ^ ((x) >> 7)) & 0x00AA00AA00AA00AALL; \ - (x) = (x) ^ (t) ^ ((t) << 7); \ - (t) = ((x) ^ ((x) >> 14)) & 0x0000CCCC0000CCCCLL; \ - (x) = (x) ^ (t) ^ ((t) << 14); \ - (t) = ((x) ^ ((x) >> 28)) & 0x00000000F0F0F0F0LL; \ - (x) = (x) ^ (t) ^ ((t) << 28); \ + t = (x ^ (x >> 7)) & 0x00AA00AA00AA00AALL; \ + x = x ^ t ^ (t << 7); \ + t = (x ^ (x >> 14)) & 0x0000CCCC0000CCCCLL; \ + x = x ^ t ^ (t << 14); \ + t = (x ^ (x >> 28)) & 0x00000000F0F0F0F0LL; \ + x = x ^ t ^ (t << 28); \ } /* Transpose 8x8 bit array along the diagonal from upper right to lower left */ #define TRANS_BIT_8X8_BE(x, t) { \ - (t) = ((x) ^ ((x) >> 9)) & 0x0055005500550055LL; \ - (x) = (x) ^ (t) ^ ((t) << 9); \ - (t) = ((x) ^ ((x) >> 18)) & 0x0000333300003333LL; \ - (x) = (x) ^ (t) ^ ((t) << 18); \ - (t) = ((x) ^ ((x) >> 36)) & 0x000000000F0F0F0FLL; \ - (x) = (x) ^ (t) ^ ((t) << 36); \ + t = (x ^ (x >> 9)) & 0x0055005500550055LL; \ + x = x ^ t ^ (t << 9); \ + t = (x ^ (x >> 18)) & 0x0000333300003333LL; \ + x = x ^ t ^ (t << 18); \ + t = (x ^ (x >> 36)) & 0x000000000F0F0F0FLL; \ + x = x ^ t ^ (t << 36); \ } /* Transpose of an array of arbitrarily typed elements. */ #define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \ - type_t* in_type = (type_t*) (in); \ - type_t* out_type = (type_t*) (out); \ size_t ii, jj, kk; \ - for (ii = 0; ii + 7 < (lda); ii += 8) { \ - for (jj = 0; jj < (ldb); jj++) { \ - for (kk = 0; kk < 8; kk++) { \ - out_type[jj*(lda) + ii + kk] = \ - in_type[ii*(ldb) + kk * (ldb) + jj]; \ + const type_t* in_type = (const type_t*) in; \ + type_t* out_type = (type_t*) out; \ + for(ii = 0; ii + 7 < lda; ii += 8) { \ + for(jj = 0; jj < ldb; jj++) { \ + for(kk = 0; kk < 8; kk++) { \ + out_type[jj*lda + ii + kk] = \ + in_type[ii*ldb + kk * ldb + jj]; \ } \ } \ } \ - for (ii = (lda) - (lda) % 8; ii < (lda); ii ++) { \ - for (jj = 0; jj < (ldb); jj++) { \ - out_type[jj*(lda) + ii] = in_type[ii*(ldb) + jj]; \ + for(ii = lda - lda % 8; ii < lda; ii ++) { \ + for(jj = 0; jj < ldb; jj++) { \ + out_type[jj*lda + ii] = in_type[ii*ldb + jj]; \ } \ } \ } @@ -158,9 +169,4 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); - -#ifdef __cplusplus -} -#endif - -#endif /* BITSHUFFLE_GENERIC_H */ +#endif /* BLOSC_BITSHUFFLE_GENERIC_H */ diff --git a/src/c-blosc2/blosc/bitshuffle-neon.c b/src/c-blosc2/blosc/bitshuffle-neon.c index adc7eec6..8b1b5948 100644 --- a/src/c-blosc2/blosc/bitshuffle-neon.c +++ b/src/c-blosc2/blosc/bitshuffle-neon.c @@ -16,6 +16,8 @@ #include +#include + /* The next is useful for debugging purposes */ #if 0 #include diff --git a/src/c-blosc2/blosc/bitshuffle-neon.h b/src/c-blosc2/blosc/bitshuffle-neon.h index b8325c1f..5d9f04af 100644 --- a/src/c-blosc2/blosc/bitshuffle-neon.h +++ b/src/c-blosc2/blosc/bitshuffle-neon.h @@ -12,14 +12,13 @@ /* NEON-accelerated bitshuffle/bitunshuffle routines. */ -#ifndef BITSHUFFLE_NEON_H -#define BITSHUFFLE_NEON_H +#ifndef BLOSC_BITSHUFFLE_NEON_H +#define BLOSC_BITSHUFFLE_NEON_H #include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif +#include +#include /** NEON-accelerated bitshuffle routine. @@ -33,8 +32,4 @@ BLOSC_NO_EXPORT int64_t bitshuffle_neon(void* _src, void* _dest, const size_t bl BLOSC_NO_EXPORT int64_t bitunshuffle_neon(void* _src, void* _dest, const size_t blocksize, const size_t bytesoftype, void* tmp_buf); -#ifdef __cplusplus -} -#endif - -#endif /* BITSHUFFLE_NEON_H */ +#endif /* BLOSC_BITSHUFFLE_NEON_H */ diff --git a/src/c-blosc2/blosc/bitshuffle-sse2.c b/src/c-blosc2/blosc/bitshuffle-sse2.c index 8314ea22..ed5769de 100644 --- a/src/c-blosc2/blosc/bitshuffle-sse2.c +++ b/src/c-blosc2/blosc/bitshuffle-sse2.c @@ -29,6 +29,8 @@ #include +#include + /* The next is useful for debugging purposes */ #if 0 #include diff --git a/src/c-blosc2/blosc/bitshuffle-sse2.h b/src/c-blosc2/blosc/bitshuffle-sse2.h index d3d53049..2d31789d 100644 --- a/src/c-blosc2/blosc/bitshuffle-sse2.h +++ b/src/c-blosc2/blosc/bitshuffle-sse2.h @@ -10,15 +10,13 @@ /* SSE2-accelerated shuffle/unshuffle routines. */ -#ifndef BITSHUFFLE_SSE2_H -#define BITSHUFFLE_SSE2_H +#ifndef BLOSC_BITSHUFFLE_SSE2_H +#define BLOSC_BITSHUFFLE_SSE2_H #include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif - +#include +#include BLOSC_NO_EXPORT int64_t bshuf_trans_byte_elem_sse2(void* in, void* out, const size_t size, @@ -46,9 +44,4 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); -#ifdef __cplusplus -} -#endif - - -#endif /* BITSHUFFLE_SSE2_H */ +#endif /* BLOSC_BITSHUFFLE_SSE2_H */ diff --git a/src/c-blosc2/blosc/blosc-private.h b/src/c-blosc2/blosc/blosc-private.h index 1fba14a3..64097f07 100644 --- a/src/c-blosc2/blosc/blosc-private.h +++ b/src/c-blosc2/blosc/blosc-private.h @@ -9,16 +9,15 @@ **********************************************************************/ -#ifndef IARRAY_BLOSC_PRIVATE_H -#define IARRAY_BLOSC_PRIVATE_H +#ifndef BLOSC_BLOSC_PRIVATE_H +#define BLOSC_BLOSC_PRIVATE_H -#ifdef __cplusplus -extern "C" { -#endif - -#include "stdbool.h" -#include "blosc2.h" #include "blosc2/blosc2-common.h" +#include "blosc2.h" + +#include +#include +#include /********************************************************************* @@ -181,8 +180,97 @@ int fill_tuner(blosc2_tuner *tuner); extern blosc2_tuner g_tuners[256]; extern int g_ntuners; -#ifdef __cplusplus + + +#if defined(_WIN32) +#include +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif +#define RTLD_LAZY 0x000 +#define popen _popen +#define pclose _pclose + +static struct { + long lasterror; + const char *err_rutin; +} +var = { + 0, + NULL +}; + +static inline void *dlopen (const char *filename, int flags) { + HINSTANCE hInst; + hInst = LoadLibrary(filename); + if (hInst==NULL) { + var.lasterror = GetLastError(); + var.err_rutin = "dlopen"; + } + + return hInst; +} + +static inline void *dlsym(void *handle, const char *name) { + FARPROC fp; + fp = GetProcAddress((HINSTANCE)handle, name); + if (!fp) { + var.lasterror = GetLastError (); + var.err_rutin = "dlsym"; + } + return (void *)(intptr_t)fp; +} + +static inline int dlclose(void *handle) { + bool ok = FreeLibrary((HINSTANCE)handle); + if (!ok) { + var.lasterror = GetLastError(); + var.err_rutin = "dlclose"; + return BLOSC2_ERROR_FAILURE; + } + return BLOSC2_ERROR_SUCCESS; } + +static inline const char *dlerror (void) { + static char errstr [88]; + if (var.lasterror) { + sprintf (errstr, "%s error #%ld", var.err_rutin, var.lasterror); + return errstr; + } else { + return NULL; + } +} +#else +#include #endif -#endif //IARRAY_BLOSC_PRIVATE_H + +static inline void* load_lib(char *plugin_name, char *libpath) { + char python_cmd[PATH_MAX] = {0}; + sprintf(python_cmd, "python -c \"import blosc2_%s; blosc2_%s.print_libpath()\"", plugin_name, plugin_name); + FILE *fp = popen(python_cmd, "r"); + if (fp == NULL) { + BLOSC_TRACE_ERROR("Could not run python"); + return NULL; + } + if (fgets(libpath, PATH_MAX, fp) == NULL) { + BLOSC_TRACE_ERROR("Could not read python output"); + pclose(fp); + return NULL; + } + pclose(fp); + if (strlen(libpath) == 0) { + BLOSC_TRACE_ERROR("Could not find plugin libpath"); + return NULL; + } + void* loaded_lib; + BLOSC_TRACE_INFO("libpath for plugin blosc2_%s: %s\n", plugin_name, libpath); + loaded_lib = dlopen(libpath, RTLD_LAZY); + if (loaded_lib == NULL) { + BLOSC_TRACE_ERROR("Attempt to load plugin in path '%s' failed with error: %s", + libpath, dlerror()); + } + return loaded_lib; +} + +#endif /* BLOSC_BLOSC_PRIVATE_H */ diff --git a/src/c-blosc2/blosc/blosc2-stdio.c b/src/c-blosc2/blosc/blosc2-stdio.c index b5227db3..912c02d3 100644 --- a/src/c-blosc2/blosc/blosc2-stdio.c +++ b/src/c-blosc2/blosc/blosc2-stdio.c @@ -12,6 +12,10 @@ #include "blosc2/blosc2-stdio.h" #include "blosc2.h" +#include +#include +#include + void *blosc2_stdio_open(const char *urlpath, const char *mode, void *params) { BLOSC_UNUSED_PARAM(params); FILE *file = fopen(urlpath, mode); diff --git a/src/c-blosc2/blosc/blosc2.c b/src/c-blosc2/blosc/blosc2.c index 785ee171..1b691f0e 100644 --- a/src/c-blosc2/blosc/blosc2.c +++ b/src/c-blosc2/blosc/blosc2.c @@ -12,7 +12,6 @@ #include "blosc2.h" #include "blosc-private.h" #include "../plugins/codecs/zfp/blosc2-zfp.h" -#include "blosc2/plugins-utils.h" #include "frame.h" #if defined(USING_CMAKE) @@ -108,46 +107,54 @@ int release_threadpool(blosc2_context *context); /* Wait until all threads are initialized */ #ifdef BLOSC_POSIX_BARRIERS -#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ - rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_init); \ - if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ - BLOSC_TRACE_ERROR("Could not wait on barrier (init): %d", rc); \ - return((RET_VAL)); \ - } +#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ + do { \ + rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_init); \ + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ + BLOSC_TRACE_ERROR("Could not wait on barrier (init): %d", rc); \ + return((RET_VAL)); \ + } \ + } while (0) #else -#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ - pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ - if ((CONTEXT_PTR)->count_threads < (CONTEXT_PTR)->nthreads) { \ - (CONTEXT_PTR)->count_threads++; \ - pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ - &(CONTEXT_PTR)->count_threads_mutex); \ - } \ - else { \ - pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ - } \ - pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); +#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ + do { \ + pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ + if ((CONTEXT_PTR)->count_threads < (CONTEXT_PTR)->nthreads) { \ + (CONTEXT_PTR)->count_threads++; \ + pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ + &(CONTEXT_PTR)->count_threads_mutex); \ + } \ + else { \ + pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ + } \ + pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); \ + } while (0) #endif /* Wait for all threads to finish */ #ifdef BLOSC_POSIX_BARRIERS -#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ - rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_finish); \ - if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ - BLOSC_TRACE_ERROR("Could not wait on barrier (finish)"); \ - return((RET_VAL)); \ - } +#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ + do { \ + rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_finish); \ + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ + BLOSC_TRACE_ERROR("Could not wait on barrier (finish)"); \ + return((RET_VAL)); \ + } \ + } while (0) #else -#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ - pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ - if ((CONTEXT_PTR)->count_threads > 0) { \ - (CONTEXT_PTR)->count_threads--; \ - pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ - &(CONTEXT_PTR)->count_threads_mutex); \ - } \ - else { \ - pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ - } \ - pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); +#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ + do { \ + pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ + if ((CONTEXT_PTR)->count_threads > 0) { \ + (CONTEXT_PTR)->count_threads--; \ + pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ + &(CONTEXT_PTR)->count_threads_mutex); \ + } \ + else { \ + pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ + } \ + pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); \ + } while (0) #endif @@ -816,16 +823,21 @@ int fill_codec(blosc2_codec *codec) { char libpath[PATH_MAX]; void *lib = load_lib(codec->compname, libpath); if(lib == NULL) { - BLOSC_TRACE_ERROR("Error while loading the library"); + BLOSC_TRACE_ERROR("Error while loading the library for codec `%s`", codec->compname); return BLOSC2_ERROR_FAILURE; } codec_info *info = dlsym(lib, "info"); + if (info == NULL) { + BLOSC_TRACE_ERROR("`info` symbol cannot be loaded from plugin `%s`", codec->compname); + dlclose(lib); + return BLOSC2_ERROR_FAILURE; + } + codec->encoder = dlsym(lib, info->encoder); codec->decoder = dlsym(lib, info->decoder); - - if (codec->encoder == NULL || codec->decoder == NULL){ - BLOSC_TRACE_ERROR("Wrong library loaded"); + if (codec->encoder == NULL || codec->decoder == NULL) { + BLOSC_TRACE_ERROR("encoder or decoder cannot be loaded from plugin `%s`", codec->compname); dlclose(lib); return BLOSC2_ERROR_FAILURE; } @@ -900,6 +912,12 @@ static int blosc2_intialize_header_from_context(blosc2_context* context, blosc_h return 0; } +void _cycle_buffers(uint8_t **src, uint8_t **dest, uint8_t **tmp) { + uint8_t *tmp2 = *src; + *src = *dest; + *dest = *tmp; + *tmp = tmp2; +} uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t bsize, const uint8_t* src, const int32_t offset, @@ -941,10 +959,7 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b // No more filters are required return _dest; } - // Cycle buffers - _src = _dest; - _dest = _tmp; - _tmp = _src; + _cycle_buffers(&_src, &_dest, &_tmp); } /* Process the filter pipeline */ @@ -957,9 +972,7 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b shuffle(typesize, bsize, _src, _dest); // Cycle filters when required if (j < filters_meta[i]) { - _src = _dest; - _dest = _tmp; - _tmp = _src; + _cycle_buffers(&_src, &_dest, &_tmp); } } break; @@ -990,7 +1003,7 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b if (g_filters[j].forward == NULL) { // Dynamically load library if (fill_filter(&g_filters[j]) < 0) { - BLOSC_TRACE_ERROR("Could not load filter %d \n", g_filters[j].id); + BLOSC_TRACE_ERROR("Could not load filter %d\n", g_filters[j].id); return NULL; } } @@ -1018,9 +1031,7 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b // Cycle buffers when required if (filters[i] != BLOSC_NOFILTER) { - _src = _dest; - _dest = _tmp; - _tmp = _src; + _cycle_buffers(&_src, &_dest, &_tmp); } } return _src; @@ -1180,7 +1191,7 @@ static int blosc_c(struct thread_context* thread_context, int32_t bsize, } maxout = neblock; - if (ntbytes + maxout > destsize) { + if (ntbytes + maxout > destsize && !instr_codec) { /* avoid buffer * overrun */ maxout = destsize - ntbytes; if (maxout <= 0) { @@ -1337,9 +1348,7 @@ int pipeline_backward(struct thread_context* thread_context, const int32_t bsize unshuffle(typesize, bsize, _src, _dest); // Cycle filters when required if (j < filters_meta[i]) { - _src = _dest; - _dest = _tmp; - _tmp = _src; + _cycle_buffers(&_src, &_dest, &_tmp); } // Check whether we have to copy the intermediate _dest buffer to final destination if (last_copy_filter && (filters_meta[i] % 2) == 1 && j == filters_meta[i]) { @@ -1417,9 +1426,7 @@ int pipeline_backward(struct thread_context* thread_context, const int32_t bsize // Cycle buffers when required if ((filters[i] != BLOSC_NOFILTER) && (filters[i] != BLOSC_TRUNC_PREC)) { - _src = _dest; - _dest = _tmp; - _tmp = _src; + _cycle_buffers(&_src, &_dest, &_tmp); } if (last_filter_index == i) { break; @@ -1917,7 +1924,7 @@ static int serial_blosc(struct thread_context* thread_context) { blosc2_context* context = thread_context->parent_context; int32_t j, bsize, leftoverblock; int32_t cbytes; - int32_t ntbytes = (int32_t)context->output_bytes; + int32_t ntbytes = context->output_bytes; int32_t* bstarts = context->bstarts; uint8_t* tmp = thread_context->tmp; uint8_t* tmp2 = thread_context->tmp2; @@ -4569,24 +4576,25 @@ blosc2_io_cb *blosc2_get_io_cb(uint8_t id) { } void blosc2_unidim_to_multidim(uint8_t ndim, int64_t *shape, int64_t i, int64_t *index) { - int64_t strides[BLOSC2_MAX_DIM]; if (ndim == 0) { return; } + int64_t *strides = malloc(ndim * sizeof(int64_t)); strides[ndim - 1] = 1; - for (int j = ndim - 2; j >= 0; --j) { - strides[j] = shape[j + 1] * strides[j + 1]; - } + for (int j = ndim - 2; j >= 0; --j) { + strides[j] = shape[j + 1] * strides[j + 1]; + } - index[0] = i / strides[0]; - for (int j = 1; j < ndim; ++j) { - index[j] = (i % strides[j - 1]) / strides[j]; - } + index[0] = i / strides[0]; + for (int j = 1; j < ndim; ++j) { + index[j] = (i % strides[j - 1]) / strides[j]; + } + free(strides); } void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, const int64_t *strides, int64_t *i) { - *i = 0; - for (int j = 0; j < ndim; ++j) { - *i += index[j] * strides[j]; - } + *i = 0; + for (int j = 0; j < ndim; ++j) { + *i += index[j] * strides[j]; + } } diff --git a/src/c-blosc2/blosc/blosclz.c b/src/c-blosc2/blosc/blosclz.c index 283f5130..00eda33b 100644 --- a/src/c-blosc2/blosc/blosclz.c +++ b/src/c-blosc2/blosc/blosclz.c @@ -18,9 +18,10 @@ #include "fastcopy.h" #include "blosc2/blosc2-common.h" -#include #include - +#include +#include +#include /* * Give hints to the compiler for branch prediction optimization. diff --git a/src/c-blosc2/blosc/blosclz.h b/src/c-blosc2/blosc/blosclz.h index f91633eb..dfe49bf9 100644 --- a/src/c-blosc2/blosc/blosclz.h +++ b/src/c-blosc2/blosc/blosclz.h @@ -14,18 +14,13 @@ about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_BLOSCLZ_H +#define BLOSC_BLOSCLZ_H -#ifndef BLOSCLZ_H -#define BLOSCLZ_H #include "context.h" -#if defined (__cplusplus) -extern "C" { -#endif - #define BLOSCLZ_VERSION_STRING "2.5.3" - /** Compress a block of data in the input buffer and returns the size of compressed block. The size of input buffer is specified by @@ -63,8 +58,4 @@ int blosclz_compress(int opt_level, const void* input, int length, int blosclz_decompress(const void* input, int length, void* output, int maxout); -#if defined (__cplusplus) -} -#endif - -#endif /* BLOSCLZ_H */ +#endif /* BLOSC_BLOSCLZ_H */ diff --git a/src/c-blosc2/blosc/context.h b/src/c-blosc2/blosc/context.h index fc168511..4941df99 100644 --- a/src/c-blosc2/blosc/context.h +++ b/src/c-blosc2/blosc/context.h @@ -8,32 +8,35 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef CONTEXT_H -#define CONTEXT_H +#ifndef BLOSC_CONTEXT_H +#define BLOSC_CONTEXT_H + +#include "b2nd.h" +#include "blosc2.h" + +#if defined(HAVE_ZSTD) +#include "zstd.h" +#endif + +#ifdef HAVE_IPP +#include +#endif #if defined(_WIN32) && !defined(__GNUC__) - #include "win32/pthread.h" +#include "win32/pthread.h" #else - #include +#include #endif +#include +#include + /* Have problems using posix barriers when symbol value is 200112L */ /* Requires more investigation, but this will work for the moment */ #if defined(_POSIX_BARRIERS) && ( (_POSIX_BARRIERS - 20012L) >= 0 && _POSIX_BARRIERS != 200112L) #define BLOSC_POSIX_BARRIERS #endif -#include "blosc2.h" -#include "b2nd.h" - -#if defined(HAVE_ZSTD) - #include "zstd.h" -#endif /* HAVE_ZSTD */ - -#ifdef HAVE_IPP - #include -#endif /* HAVE_IPP */ - struct blosc2_context_s { const uint8_t* src; /* The source buffer */ uint8_t* dest; /* The destination buffer */ @@ -148,5 +151,4 @@ struct thread_context { #endif }; - -#endif /* CONTEXT_H */ +#endif /* BLOSC_CONTEXT_H */ diff --git a/src/c-blosc2/blosc/delta.c b/src/c-blosc2/blosc/delta.c index 2d622ae2..adddfbff 100644 --- a/src/c-blosc2/blosc/delta.c +++ b/src/c-blosc2/blosc/delta.c @@ -11,6 +11,7 @@ #include "delta.h" #include +#include /* Apply the delta filters to src. This can never fail. */ diff --git a/src/c-blosc2/blosc/delta.h b/src/c-blosc2/blosc/delta.h index 082b07c5..2e9ce9a8 100644 --- a/src/c-blosc2/blosc/delta.h +++ b/src/c-blosc2/blosc/delta.h @@ -11,7 +11,6 @@ #ifndef BLOSC_DELTA_H #define BLOSC_DELTA_H -#include #include void delta_encoder(const uint8_t* dref, int32_t offset, int32_t nbytes, @@ -20,4 +19,4 @@ void delta_encoder(const uint8_t* dref, int32_t offset, int32_t nbytes, void delta_decoder(const uint8_t* dref, int32_t offset, int32_t nbytes, int32_t typesize, uint8_t* dest); -#endif //BLOSC_DELTA_H +#endif /* BLOSC_DELTA_H */ diff --git a/src/c-blosc2/blosc/directories.c b/src/c-blosc2/blosc/directories.c index 15f2505c..13a3ac0f 100644 --- a/src/c-blosc2/blosc/directories.c +++ b/src/c-blosc2/blosc/directories.c @@ -10,8 +10,12 @@ #include "blosc2.h" -#include #include +#include + +#include +#include +#include #if defined(_WIN32) || defined(__MINGW32__) #include @@ -127,6 +131,10 @@ int blosc2_remove_urlpath(const char* urlpath){ if (urlpath != NULL) { struct stat statbuf; if (stat(urlpath, &statbuf) != 0){ + if (errno == ENOENT) { + // Path does not exist + return BLOSC2_ERROR_SUCCESS; + } BLOSC_TRACE_ERROR("Could not access %s", urlpath); return BLOSC2_ERROR_FAILURE; } diff --git a/src/c-blosc2/blosc/fastcopy.c b/src/c-blosc2/blosc/fastcopy.c index 9ff6d229..81e866c3 100644 --- a/src/c-blosc2/blosc/fastcopy.c +++ b/src/c-blosc2/blosc/fastcopy.c @@ -21,6 +21,10 @@ #include "blosc2/blosc2-common.h" #include +#include +#if defined(BLOSC_STRICT_ALIGN) +#include +#endif /* * Use inlined functions for supported systems. diff --git a/src/c-blosc2/blosc/fastcopy.h b/src/c-blosc2/blosc/fastcopy.h index 0667aa84..ae2c27ac 100644 --- a/src/c-blosc2/blosc/fastcopy.h +++ b/src/c-blosc2/blosc/fastcopy.h @@ -17,4 +17,4 @@ unsigned char *fastcopy(unsigned char *out, const unsigned char *from, unsigned /* Same as fastcopy() but without overwriting origin or destination when they overlap */ unsigned char* copy_match(unsigned char *out, const unsigned char *from, unsigned len); -#endif //BLOSC_FASTCOPY_H +#endif /* BLOSC_FASTCOPY_H */ diff --git a/src/c-blosc2/blosc/frame.c b/src/c-blosc2/blosc/frame.c index 1a437e89..e5e0a30a 100644 --- a/src/c-blosc2/blosc/frame.c +++ b/src/c-blosc2/blosc/frame.c @@ -19,18 +19,19 @@ #include #endif /* _WIN32 */ -/* If C11 is supported, use it's built-in aligned allocation. */ -#if __STDC_VERSION__ >= 201112L -#include -#endif +#include +#include +#include #include +#include #include #include -#include -#include -#include -#include + +/* If C11 is supported, use it's built-in aligned allocation. */ +#if __STDC_VERSION__ >= 201112L +#include +#endif /* Create a new (empty) frame */ @@ -1345,7 +1346,7 @@ static int get_meta_from_header(blosc2_frame_s* frame, blosc2_schunk* schunk, ui if ((*idxp & 0xe0u) != 0xa0u) { // sanity check return BLOSC2_ERROR_DATA; } - blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1); + blosc2_metalayer* metalayer = calloc(1, sizeof(blosc2_metalayer)); schunk->metalayers[nmetalayer] = metalayer; // Populate the metalayer string @@ -1525,7 +1526,7 @@ static int get_vlmeta_from_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk, if ((*idxp & 0xe0u) != 0xa0u) { // sanity check return BLOSC2_ERROR_DATA; } - blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1); + blosc2_metalayer* metalayer = calloc(1, sizeof(blosc2_metalayer)); schunk->vlmetalayers[nmetalayer] = metalayer; // Populate the metalayer string diff --git a/src/c-blosc2/blosc/frame.h b/src/c-blosc2/blosc/frame.h index b6ca3775..20e76878 100644 --- a/src/c-blosc2/blosc/frame.h +++ b/src/c-blosc2/blosc/frame.h @@ -13,9 +13,8 @@ #include "blosc2.h" -#include -#include #include +#include // Different types of frames #define FRAME_CONTIGUOUS_TYPE 0 @@ -168,4 +167,4 @@ int frame_update_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk); int64_t frame_fill_special(blosc2_frame_s* frame, int64_t nitems, int special_value, int32_t chunksize, blosc2_schunk* schunk); -#endif //BLOSC_FRAME_H +#endif /* BLOSC_FRAME_H */ diff --git a/src/c-blosc2/blosc/schunk.c b/src/c-blosc2/blosc/schunk.c index 7ef3336c..4bafddfd 100644 --- a/src/c-blosc2/blosc/schunk.c +++ b/src/c-blosc2/blosc/schunk.c @@ -8,27 +8,26 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ - -#include "blosc-private.h" -#include "blosc2/tuners-registry.h" #include "frame.h" #include "stune.h" +#include "blosc-private.h" +#include "blosc2/tuners-registry.h" #include "blosc2.h" #if defined(_WIN32) #include #include #include - #define mkdir(D, M) _mkdir(D) #endif /* _WIN32 */ +#include + +#include #include +#include #include #include -#include -#include -#include /* If C11 is supported, use it's built-in aligned allocation. */ #if __STDC_VERSION__ >= 201112L @@ -38,7 +37,7 @@ /* Get the cparams associated with a super-chunk */ int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams) { - *cparams = calloc(sizeof(blosc2_cparams), 1); + *cparams = calloc(1, sizeof(blosc2_cparams)); (*cparams)->schunk = schunk; for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { (*cparams)->filters[i] = schunk->filters[i]; @@ -62,7 +61,7 @@ int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams) { /* Get the dparams associated with a super-chunk */ int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams) { - *dparams = calloc(sizeof(blosc2_dparams), 1); + *dparams = calloc(1, sizeof(blosc2_dparams)); (*dparams)->schunk = schunk; if (schunk->dctx == NULL) { (*dparams)->nthreads = blosc2_get_nthreads(); @@ -91,7 +90,9 @@ void update_schunk_properties(struct blosc2_schunk* schunk) { schunk->chunksize = -1; schunk->tuner_params = cparams->tuner_params; schunk->tuner_id = cparams->tuner_id; - + if (cparams->tuner_id == BLOSC_BTUNE) { + cparams->use_dict = 0; + } /* The compression context */ if (schunk->cctx != NULL) { blosc2_free_ctx(schunk->cctx); @@ -124,9 +125,9 @@ blosc2_schunk* blosc2_schunk_new(blosc2_storage *storage) { // Update the (local variable) storage storage = schunk->storage; - char* btune_balance = getenv("BTUNE_BALANCE"); - if (btune_balance != NULL) { - // If BTUNE_BALANCE passed, automatically use btune + char* tradeoff = getenv("BTUNE_TRADEOFF"); + if (tradeoff != NULL) { + // If BTUNE_TRADEOFF passed, automatically use btune storage->cparams->tuner_id = BLOSC_BTUNE; } @@ -343,7 +344,7 @@ blosc2_schunk* blosc2_schunk_open(const char* urlpath) { return blosc2_schunk_open_udio(urlpath, &BLOSC2_IO_DEFAULTS); } -BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open_offset(const char* urlpath, int64_t offset) { +blosc2_schunk* blosc2_schunk_open_offset(const char* urlpath, int64_t offset) { if (urlpath == NULL) { BLOSC_TRACE_ERROR("You need to supply a urlpath."); return NULL; diff --git a/src/c-blosc2/blosc/sframe.c b/src/c-blosc2/blosc/sframe.c index b7dacaa4..efac04e0 100644 --- a/src/c-blosc2/blosc/sframe.c +++ b/src/c-blosc2/blosc/sframe.c @@ -12,6 +12,7 @@ #include "blosc2.h" #include +#include #include #include diff --git a/src/c-blosc2/blosc/sframe.h b/src/c-blosc2/blosc/sframe.h index 0cabc0c4..b62979ff 100644 --- a/src/c-blosc2/blosc/sframe.h +++ b/src/c-blosc2/blosc/sframe.h @@ -11,10 +11,15 @@ #ifndef BLOSC_SFRAME_H #define BLOSC_SFRAME_H +#include "frame.h" + +#include +#include + void* sframe_open_index(const char* urlpath, const char* mode, const blosc2_io *io); void* sframe_open_chunk(const char* urlpath, int64_t nchunk, const char* mode, const blosc2_io *io); int sframe_delete_chunk(const char* urlpath, int64_t nchunk); void* sframe_create_chunk(blosc2_frame_s* frame, uint8_t* chunk, int64_t nchunk, int64_t cbytes); int32_t sframe_get_chunk(blosc2_frame_s* frame, int64_t nchunk, uint8_t** chunk, bool* needs_free); -#endif //BLOSC_SFRAME_H +#endif /* BLOSC_SFRAME_H */ diff --git a/src/c-blosc2/blosc/shuffle-altivec.c b/src/c-blosc2/blosc/shuffle-altivec.c index 1f92d5bb..d0077c50 100644 --- a/src/c-blosc2/blosc/shuffle-altivec.c +++ b/src/c-blosc2/blosc/shuffle-altivec.c @@ -14,9 +14,12 @@ /* Make sure ALTIVEC is available for the compilation target and compiler. */ #if defined(__ALTIVEC__) -#include #include "transpose-altivec.h" +#include + +#include + /* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ static void shuffle2_altivec(uint8_t* const dest, const uint8_t* const src, diff --git a/src/c-blosc2/blosc/shuffle-altivec.h b/src/c-blosc2/blosc/shuffle-altivec.h index 1d46a09a..1fd98a0f 100644 --- a/src/c-blosc2/blosc/shuffle-altivec.h +++ b/src/c-blosc2/blosc/shuffle-altivec.h @@ -10,14 +10,12 @@ /* ALTIVEC-accelerated shuffle/unshuffle routines. */ -#ifndef SHUFFLE_ALTIVEC_H -#define SHUFFLE_ALTIVEC_H +#ifndef BLOSC_SHUFFLE_ALTIVEC_H +#define BLOSC_SHUFFLE_ALTIVEC_H #include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif +#include /** ALTIVEC-accelerated shuffle routine. @@ -31,8 +29,4 @@ BLOSC_NO_EXPORT void shuffle_altivec(const int32_t bytesoftype, const int32_t bl BLOSC_NO_EXPORT void unshuffle_altivec(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#ifdef __cplusplus -} -#endif - -#endif /* SHUFFLE_ALTIVEC_H */ +#endif /* BLOSC_SHUFFLE_ALTIVEC_H */ diff --git a/src/c-blosc2/blosc/shuffle-avx2.c b/src/c-blosc2/blosc/shuffle-avx2.c index 800bb2d0..efb10bf2 100644 --- a/src/c-blosc2/blosc/shuffle-avx2.c +++ b/src/c-blosc2/blosc/shuffle-avx2.c @@ -16,6 +16,8 @@ #include +#include +#include /* The next is useful for debugging purposes */ #if 0 diff --git a/src/c-blosc2/blosc/shuffle-avx2.h b/src/c-blosc2/blosc/shuffle-avx2.h index 535493fd..1159ad53 100644 --- a/src/c-blosc2/blosc/shuffle-avx2.h +++ b/src/c-blosc2/blosc/shuffle-avx2.h @@ -15,9 +15,7 @@ #include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif +#include /** AVX2-accelerated shuffle routine. @@ -31,8 +29,4 @@ BLOSC_NO_EXPORT void shuffle_avx2(const int32_t bytesoftype, const int32_t block BLOSC_NO_EXPORT void unshuffle_avx2(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#ifdef __cplusplus -} -#endif - #endif /* SHUFFLE_AVX2_H */ diff --git a/src/c-blosc2/blosc/shuffle-generic.h b/src/c-blosc2/blosc/shuffle-generic.h index 22218d8e..5c2e0126 100644 --- a/src/c-blosc2/blosc/shuffle-generic.h +++ b/src/c-blosc2/blosc/shuffle-generic.h @@ -16,16 +16,13 @@ which isn't a multiple of the hardware's vector size. **********************************************************************/ - -#ifndef SHUFFLE_GENERIC_H -#define SHUFFLE_GENERIC_H +#ifndef BLOSC_SHUFFLE_GENERIC_H +#define BLOSC_SHUFFLE_GENERIC_H #include "blosc2/blosc2-common.h" -#include -#ifdef __cplusplus -extern "C" { -#endif +#include +#include /** Generic (non-hardware-accelerated) shuffle routine. @@ -34,7 +31,7 @@ extern "C" { implementations to process any remaining elements in a block which is not a multiple of (type_size * vector_size). */ -inline static void shuffle_generic_inline(const int32_t type_size, +static inline void shuffle_generic_inline(const int32_t type_size, const int32_t vectorizable_blocksize, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest) { int32_t i, j; @@ -62,7 +59,7 @@ inline static void shuffle_generic_inline(const int32_t type_size, implementations to process any remaining elements in a block which is not a multiple of (type_size * vector_size). */ -inline static void unshuffle_generic_inline(const int32_t type_size, +static inline void unshuffle_generic_inline(const int32_t type_size, const int32_t vectorizable_blocksize, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest) { int32_t i, j; @@ -95,8 +92,4 @@ BLOSC_NO_EXPORT void shuffle_generic(const int32_t bytesoftype, const int32_t bl BLOSC_NO_EXPORT void unshuffle_generic(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#ifdef __cplusplus -} -#endif - -#endif /* SHUFFLE_GENERIC_H */ +#endif /* BLOSC_SHUFFLE_GENERIC_H */ diff --git a/src/c-blosc2/blosc/shuffle-neon.c b/src/c-blosc2/blosc/shuffle-neon.c index 7dcfeff5..c1940c1f 100644 --- a/src/c-blosc2/blosc/shuffle-neon.c +++ b/src/c-blosc2/blosc/shuffle-neon.c @@ -17,6 +17,7 @@ #include +#include /* The next is useful for debugging purposes */ #if 0 diff --git a/src/c-blosc2/blosc/shuffle-neon.h b/src/c-blosc2/blosc/shuffle-neon.h index 50c489e2..b7cb6a25 100644 --- a/src/c-blosc2/blosc/shuffle-neon.h +++ b/src/c-blosc2/blosc/shuffle-neon.h @@ -12,14 +12,12 @@ /* NEON-accelerated shuffle/unshuffle routines. */ -#ifndef SHUFFLE_NEON_H -#define SHUFFLE_NEON_H +#ifndef BLOSC_SHUFFLE_NEON_H +#define BLOSC_SHUFFLE_NEON_H #include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif +#include /** NEON-accelerated shuffle routine. @@ -33,8 +31,4 @@ BLOSC_NO_EXPORT void shuffle_neon(const int32_t bytesoftype, const int32_t block BLOSC_NO_EXPORT void unshuffle_neon(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#ifdef __cplusplus -} -#endif - -#endif /* SHUFFLE_NEON_H */ +#endif /* BLOSC_SHUFFLE_NEON_H */ diff --git a/src/c-blosc2/blosc/shuffle-sse2.c b/src/c-blosc2/blosc/shuffle-sse2.c index a34755ff..deee4a48 100644 --- a/src/c-blosc2/blosc/shuffle-sse2.c +++ b/src/c-blosc2/blosc/shuffle-sse2.c @@ -16,6 +16,7 @@ #include +#include /* The next is useful for debugging purposes */ #if 0 diff --git a/src/c-blosc2/blosc/shuffle-sse2.h b/src/c-blosc2/blosc/shuffle-sse2.h index 263d6705..9962b3fe 100644 --- a/src/c-blosc2/blosc/shuffle-sse2.h +++ b/src/c-blosc2/blosc/shuffle-sse2.h @@ -10,14 +10,12 @@ /* SSE2-accelerated shuffle/unshuffle routines. */ -#ifndef SHUFFLE_SSE2_H -#define SHUFFLE_SSE2_H +#ifndef BLOSC_SHUFFLE_SSE2_H +#define BLOSC_SHUFFLE_SSE2_H #include "blosc2/blosc2-common.h" -#ifdef __cplusplus -extern "C" { -#endif +#include /** SSE2-accelerated shuffle routine. @@ -31,8 +29,4 @@ BLOSC_NO_EXPORT void shuffle_sse2(const int32_t bytesoftype, const int32_t block BLOSC_NO_EXPORT void unshuffle_sse2(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#ifdef __cplusplus -} -#endif - -#endif /* SHUFFLE_SSE2_H */ +#endif /* BLOSC_SHUFFLE_SSE2_H */ diff --git a/src/c-blosc2/blosc/shuffle.c b/src/c-blosc2/blosc/shuffle.c index d75052da..2fff396f 100644 --- a/src/c-blosc2/blosc/shuffle.c +++ b/src/c-blosc2/blosc/shuffle.c @@ -8,21 +8,7 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "shuffle.h" -#include "blosc2.h" -#include "blosc2/blosc2-common.h" -#include "shuffle-generic.h" -#include "bitshuffle-generic.h" - -#include -#include -#include - - -#if !defined(__clang__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && \ - __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) -#define HAVE_CPU_FEAT_INTRIN -#endif +#include "shuffle.h" /* needs to be included first to define macros */ /* Include hardware-accelerated shuffle/unshuffle routines based on the target architecture. Note that a target architecture may support @@ -53,6 +39,22 @@ #include "bitshuffle-altivec.h" #endif /* defined(SHUFFLE_USE_ALTIVEC) */ +#include "shuffle-generic.h" +#include "bitshuffle-generic.h" +#include "blosc2/blosc2-common.h" +#include "blosc2.h" + +#include +#include +#include +#include + + +#if !defined(__clang__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) +#define HAVE_CPU_FEAT_INTRIN +#endif + /* Define function pointer types for shuffle/unshuffle routines. */ typedef void(* shuffle_func)(const int32_t, const int32_t, const uint8_t*, const uint8_t*); diff --git a/src/c-blosc2/blosc/shuffle.h b/src/c-blosc2/blosc/shuffle.h index de710dea..24784eaa 100644 --- a/src/c-blosc2/blosc/shuffle.h +++ b/src/c-blosc2/blosc/shuffle.h @@ -16,12 +16,13 @@ these are cross-platform and future-proof. **********************************************************************/ - -#ifndef SHUFFLE_H -#define SHUFFLE_H +#ifndef BLOSC_SHUFFLE_H +#define BLOSC_SHUFFLE_H #include "blosc2/blosc2-common.h" +#include + /* Toggle hardware-accelerated routines based on SHUFFLE_*_ENABLED macros and availability on the target architecture. */ @@ -41,10 +42,6 @@ #define SHUFFLE_USE_NEON #endif -#ifdef __cplusplus -extern "C" { -#endif - /** Primary shuffle and bitshuffle routines. This function dynamically dispatches to the appropriate hardware-accelerated @@ -84,8 +81,4 @@ BLOSC_NO_EXPORT int32_t const uint8_t *_src, const uint8_t *_dest, const uint8_t *_tmp, const uint8_t format_version); -#ifdef __cplusplus -} -#endif - -#endif /* SHUFFLE_H */ +#endif /* BLOSC_SHUFFLE_H */ diff --git a/src/c-blosc2/blosc/stune.h b/src/c-blosc2/blosc/stune.h index dd3cf5ab..26400e4b 100644 --- a/src/c-blosc2/blosc/stune.h +++ b/src/c-blosc2/blosc/stune.h @@ -8,11 +8,13 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef STUNE_H -#define STUNE_H +#ifndef BLOSC_STUNE_H +#define BLOSC_STUNE_H #include "context.h" +#include + /* The size of L1 cache. 32 KB is quite common nowadays. */ #define L1 (32 * 1024) /* The size of L2 cache. 256 KB is quite common nowadays. */ @@ -33,8 +35,7 @@ void blosc_stune_update(blosc2_context * context, double ctime); void blosc_stune_free(blosc2_context * context); - /* Conditions for splitting a block before compressing with a codec. */ int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize); -#endif /* STUNE_H */ +#endif /* BLOSC_STUNE_H */ diff --git a/src/c-blosc2/blosc/timestamp.c b/src/c-blosc2/blosc/timestamp.c index d855910a..e14d07ee 100644 --- a/src/c-blosc2/blosc/timestamp.c +++ b/src/c-blosc2/blosc/timestamp.c @@ -13,6 +13,8 @@ /* System-specific high-precision timing functions. */ #if defined(_WIN32) +#include + /* Set a timestamp value to the current time. */ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { /* Ignore the return value, assume the call always succeeds. */ @@ -31,9 +33,14 @@ double blosc_elapsed_nsecs(blosc_timestamp_t start_time, #else +#include + +#if defined(__MACH__) // OS X does not have clock_gettime, use clock_get_time + +#include + /* Set a timestamp value to the current time. */ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { -#ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time clock_serv_t cclock; mach_timespec_t mts; host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); @@ -41,11 +48,17 @@ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { mach_port_deallocate(mach_task_self(), cclock); timestamp->tv_sec = mts.tv_sec; timestamp->tv_nsec = mts.tv_nsec; +} + #else + +/* Set a timestamp value to the current time. */ +void blosc_set_timestamp(blosc_timestamp_t* timestamp) { clock_gettime(CLOCK_MONOTONIC, timestamp); -#endif } +#endif + /* Given two timestamp values, return the difference in nanoseconds. */ double blosc_elapsed_nsecs(blosc_timestamp_t start_time, blosc_timestamp_t end_time) { diff --git a/src/c-blosc2/blosc/transpose-altivec.h b/src/c-blosc2/blosc/transpose-altivec.h index 31996c4f..6a8b086a 100644 --- a/src/c-blosc2/blosc/transpose-altivec.h +++ b/src/c-blosc2/blosc/transpose-altivec.h @@ -11,9 +11,9 @@ #ifndef BLOSC_TRANSPOSE_ALTIVEC_H #define BLOSC_TRANSPOSE_ALTIVEC_H -#ifdef __cplusplus -extern "C" { -#endif +#include + +#include static const __vector uint8_t even = (const __vector uint8_t) { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, @@ -113,9 +113,4 @@ static void transpose16x16(__vector uint8_t * xmm0){ } } - -#ifdef __cplusplus -} -#endif - -#endif //BLOSC_TRANSPOSE_ALTIVEC_H +#endif /* BLOSC_TRANSPOSE_ALTIVEC_H */ diff --git a/src/c-blosc2/blosc/trunc-prec.c b/src/c-blosc2/blosc/trunc-prec.c index d2fb5ab6..670316de 100644 --- a/src/c-blosc2/blosc/trunc-prec.c +++ b/src/c-blosc2/blosc/trunc-prec.c @@ -13,6 +13,7 @@ #include #include +#include #include #define BITS_MANTISSA_FLOAT 23 diff --git a/src/c-blosc2/blosc/trunc-prec.h b/src/c-blosc2/blosc/trunc-prec.h index 8aa38cec..4007463f 100644 --- a/src/c-blosc2/blosc/trunc-prec.h +++ b/src/c-blosc2/blosc/trunc-prec.h @@ -11,10 +11,9 @@ #ifndef BLOSC_TRUNC_PREC_H #define BLOSC_TRUNC_PREC_H -#include #include int truncate_precision(int8_t prec_bits, int32_t typesize, int32_t nbytes, const uint8_t* src, uint8_t* dest); -#endif //BLOSC_TRUNC_PREC_H +#endif /* BLOSC_TRUNC_PREC_H */ diff --git a/src/c-blosc2/cmake/FindZLIB_NG.cmake b/src/c-blosc2/cmake/FindZLIB_NG.cmake index 679e8d4c..a72eeb89 100644 --- a/src/c-blosc2/cmake/FindZLIB_NG.cmake +++ b/src/c-blosc2/cmake/FindZLIB_NG.cmake @@ -27,9 +27,21 @@ else(ZLIB_NG_INCLUDE_DIR AND ZLIB_NG_LIBRARIES) set(ZLIB_NG_FOUND FALSE) endif(ZLIB_NG_INCLUDE_DIR AND ZLIB_NG_LIBRARIES) +# generate CMake target if(ZLIB_NG_FOUND) message(STATUS "Found zlib-ng: ${ZLIB_NG_LIBRARIES}, ${ZLIB_NG_INCLUDE_DIR}") -endif(ZLIB_NG_FOUND) + + set(ZLIB_NG_INCLUDE_DIRS ${ZLIB_NG_INCLUDE_DIR}) + + if(NOT TARGET ZLIB_NG::ZLIB_NG) + add_library(ZLIB_NG::ZLIB_NG UNKNOWN IMPORTED) + set_target_properties(ZLIB_NG::ZLIB_NG PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_NG_INCLUDE_DIRS}") + + set_property(TARGET ZLIB_NG::ZLIB_NG APPEND PROPERTY + IMPORTED_LOCATION "${ZLIB_NG_LIBRARIES}") + endif() +endif() #[[ Copyright https://github.com/zlib-ng/minizip-ng, 2021 diff --git a/src/c-blosc2/doc/reference/b2nd.rst b/src/c-blosc2/doc/reference/b2nd.rst index 6325be73..79219cea 100644 --- a/src/c-blosc2/doc/reference/b2nd.rst +++ b/src/c-blosc2/doc/reference/b2nd.rst @@ -2,7 +2,7 @@ Blosc2 NDim =========== It contains both the data and metalayer that stores the dimensional info for the array. -Blosc2 NDim has an internal context managed that stores the different properties of each array. +Blosc2 NDim has a managed internal context that stores the different properties of each array. Context ------- diff --git a/src/c-blosc2/examples/frame_roundtrip.c b/src/c-blosc2/examples/frame_roundtrip.c index f144e962..8f6ea85b 100644 --- a/src/c-blosc2/examples/frame_roundtrip.c +++ b/src/c-blosc2/examples/frame_roundtrip.c @@ -55,7 +55,8 @@ int main(void) { goto failed; } } - printf("nbytes, cbytes for schunk: %lld, %lld \n", schunk->nbytes, schunk->cbytes); + printf("nbytes, cbytes for schunk: %lld, %lld\n", + (long long int)schunk->nbytes, (long long int)schunk->cbytes); // Check contents uint8_t *chunk; @@ -69,7 +70,7 @@ int main(void) { } int nbytes = blosc2_decompress(chunk, cbytes, dest, total_bytes); if (nbytes != total_bytes) { - printf("Error in schunk: nbytes differs (%d != %d) \n", nbytes, total_bytes); + printf("Error in schunk: nbytes differs (%d != %d)\n", nbytes, total_bytes); goto failed; } if (needs_free) { @@ -77,7 +78,7 @@ int main(void) { } for (int j = 0; j < CHUNKSIZE; j++) { if (buf[j] != dest[j]) { - printf("Error in schunk: data differs in index %d (%d != %d)! \n", j, buf[i], dest[i]); + printf("Error in schunk: data differs in index %d (%d != %d)!\n", j, buf[i], dest[i]); goto failed; } } @@ -90,14 +91,15 @@ int main(void) { if (cframe_len < 0 || !cframe_needs_free) { goto failed; } - printf("converted into a cframe of %lld bytes\n", cframe_len); + printf("converted into a cframe of %lld bytes\n", (long long int)cframe_len); // Convert back into a different schunk blosc2_schunk* schunk2 = blosc2_schunk_from_buffer(cframe, cframe_len, true); if (schunk2 == NULL) { goto failed; } - printf("nbytes, cbytes for schunk2: %lld, %lld \n", schunk2->nbytes, schunk2->cbytes); + printf("nbytes, cbytes for schunk2: %lld, %lld\n", + (long long int)schunk2->nbytes, (long long int)schunk2->cbytes); // Check contents for (int i = 0; i < NCHUNKS; i++) { @@ -108,7 +110,7 @@ int main(void) { } int nbytes = blosc2_decompress(chunk, cbytes, dest, total_bytes); if (nbytes != total_bytes) { - printf("Error in schunk2: nbytes differs (%d != %d) \n", nbytes, total_bytes); + printf("Error in schunk2: nbytes differs (%d != %d)\n", nbytes, total_bytes); goto failed; } if (needs_free) { @@ -116,7 +118,7 @@ int main(void) { } for (int j = 0; j < CHUNKSIZE; j++) { if (buf[j] != dest[j]) { - printf("Error in schunk2: data differs in index %d (%d != %d)! \n", j, buf[i], dest[i]); + printf("Error in schunk2: data differs in index %d (%d != %d)!\n", j, buf[i], dest[i]); goto failed; } } diff --git a/src/c-blosc2/images/blosc2-pipeline.png b/src/c-blosc2/images/blosc2-pipeline.png new file mode 100644 index 00000000..ca604f83 Binary files /dev/null and b/src/c-blosc2/images/blosc2-pipeline.png differ diff --git a/src/c-blosc2/images/blosc2-pipeline.svg b/src/c-blosc2/images/blosc2-pipeline.svg new file mode 100644 index 00000000..f2757de1 --- /dev/null +++ b/src/c-blosc2/images/blosc2-pipeline.svg @@ -0,0 +1,652 @@ + + + +Compression processDecompression processDataCompresseddataPrefilterFilter 1Filter 2Filter 6CodecFilter pipelineDataCompresseddataPostfilterFilter 1Filter 2Filter 6CodecFilter pipeline diff --git a/src/c-blosc2/include/b2nd.h b/src/c-blosc2/include/b2nd.h index 801f5937..0e6a43aa 100644 --- a/src/c-blosc2/include/b2nd.h +++ b/src/c-blosc2/include/b2nd.h @@ -15,8 +15,8 @@ * @author Blosc Development Team */ -#ifndef B2ND_B2ND_H_ -#define B2ND_B2ND_H_ +#ifndef BLOSC_B2ND_H +#define BLOSC_B2ND_H #ifdef __cplusplus extern "C" { @@ -198,8 +198,8 @@ BLOSC_EXPORT int b2nd_zeros(b2nd_context_t *ctx, b2nd_array_t **array); * uninitialized portions of the array. * * @param ctx The b2nd context for the new array. - * @param fill_value Default value for uninitialized portions of the array. * @param array The memory pointer where the array will be created. + * @param fill_value Default value for uninitialized portions of the array. * * @return An error code. */ @@ -603,4 +603,4 @@ static inline int b2nd_deserialize_meta( } #endif -#endif // B2ND_B2ND_H_ +#endif /* BLOSC_B2ND_H */ diff --git a/src/c-blosc2/include/blosc2.h b/src/c-blosc2/include/blosc2.h index 71f58d5e..5074e2ca 100644 --- a/src/c-blosc2/include/blosc2.h +++ b/src/c-blosc2/include/blosc2.h @@ -16,19 +16,12 @@ @author The Blosc Development Team **********************************************************************/ +#ifndef BLOSC_BLOSC2_H +#define BLOSC_BLOSC2_H -#ifndef BLOSC2_H -#define BLOSC2_H - -#ifdef __cplusplus -extern "C" { -#endif #include "blosc2/blosc2-export.h" #include "blosc2/blosc2-common.h" #include "blosc2/blosc2-stdio.h" -#ifdef __cplusplus -} -#endif #if defined(_WIN32) && !defined(__MINGW32__) #include @@ -89,11 +82,11 @@ extern "C" { /* Version numbers */ #define BLOSC2_VERSION_MAJOR 2 /* for major interface/format changes */ -#define BLOSC2_VERSION_MINOR 9 /* for minor interface/format changes */ +#define BLOSC2_VERSION_MINOR 10 /* for minor interface/format changes */ #define BLOSC2_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ -#define BLOSC2_VERSION_STRING "2.9.2" /* string version. Sync with above! */ -#define BLOSC2_VERSION_DATE "$Date:: 2023-05-18 #$" /* date version */ +#define BLOSC2_VERSION_STRING "2.10.2" /* string version. Sync with above! */ +#define BLOSC2_VERSION_DATE "$Date:: 2023-08-19 #$" /* date version */ /* The maximum number of dimensions for Blosc2 NDim arrays */ @@ -103,6 +96,7 @@ extern "C" { /* Tracing macros */ #define BLOSC_TRACE_ERROR(msg, ...) BLOSC_TRACE(error, msg, ##__VA_ARGS__) #define BLOSC_TRACE_WARNING(msg, ...) BLOSC_TRACE(warning, msg, ##__VA_ARGS__) +#define BLOSC_TRACE_INFO(msg, ...) BLOSC_TRACE(info, msg, ##__VA_ARGS__) #define BLOSC_TRACE(cat, msg, ...) \ do { \ const char *__e = getenv("BLOSC_TRACE"); \ @@ -225,7 +219,7 @@ enum { BLOSC2_GLOBAL_REGISTERED_FILTERS_START = 32, BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP = 159, //!< Blosc-registered filters must be between 32 - 159. - BLOSC2_GLOBAL_REGISTERED_FILTERS = 3, + BLOSC2_GLOBAL_REGISTERED_FILTERS = 4, //!< Number of Blosc-registered filters at the moment. BLOSC2_USER_REGISTERED_FILTERS_START = 160, BLOSC2_USER_REGISTERED_FILTERS_STOP = 255, @@ -428,6 +422,8 @@ enum { /** * @brief Error codes + * Each time an error code is added here, its corresponding message error should be added in + * print_error() */ enum { BLOSC2_ERROR_SUCCESS = 0, // #endif -#endif /* SHUFFLE_COMMON_H */ +#endif /* BLOSC_BLOSC2_BLOSC2_COMMON_H */ diff --git a/src/c-blosc2/include/blosc2/blosc2-export.h b/src/c-blosc2/include/blosc2/blosc2-export.h index 3b9e4e17..303502eb 100644 --- a/src/c-blosc2/include/blosc2/blosc2-export.h +++ b/src/c-blosc2/include/blosc2/blosc2-export.h @@ -8,8 +8,8 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_EXPORT_H -#define BLOSC_EXPORT_H +#ifndef BLOSC_BLOSC2_BLOSC2_EXPORT_H +#define BLOSC_BLOSC2_BLOSC2_EXPORT_H /* Macros for specifying exported symbols. BLOSC_EXPORT is used to decorate symbols that should be @@ -45,4 +45,4 @@ #define BLOSC_NO_EXPORT BLOSC_EXPORT #endif /* defined(BLOSC_TESTING) */ -#endif /* BLOSC_EXPORT_H */ +#endif /* BLOSC_BLOSC2_BLOSC2_EXPORT_H */ diff --git a/src/c-blosc2/include/blosc2/blosc2-stdio.h b/src/c-blosc2/include/blosc2/blosc2-stdio.h index 5969987e..0556f157 100644 --- a/src/c-blosc2/include/blosc2/blosc2-stdio.h +++ b/src/c-blosc2/include/blosc2/blosc2-stdio.h @@ -8,10 +8,8 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ - -#ifndef BLOSC_BLOSC2_STDIO_H -#define BLOSC_BLOSC2_STDIO_H - +#ifndef BLOSC_BLOSC2_BLOSC2_STDIO_H +#define BLOSC_BLOSC2_BLOSC2_STDIO_H #include "blosc2-export.h" @@ -25,6 +23,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { FILE *file; } blosc2_stdio_file; @@ -37,4 +39,8 @@ BLOSC_EXPORT int64_t blosc2_stdio_write(const void *ptr, int64_t size, int64_t n BLOSC_EXPORT int64_t blosc2_stdio_read(void *ptr, int64_t size, int64_t nitems, void *stream); BLOSC_EXPORT int blosc2_stdio_truncate(void *stream, int64_t size); -#endif //BLOSC_BLOSC2_STDIO_H +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_BLOSC2_STDIO_H */ diff --git a/src/c-blosc2/include/blosc2/codecs-registry.h b/src/c-blosc2/include/blosc2/codecs-registry.h index 12f61b76..f772a0a1 100644 --- a/src/c-blosc2/include/blosc2/codecs-registry.h +++ b/src/c-blosc2/include/blosc2/codecs-registry.h @@ -8,6 +8,15 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_BLOSC2_CODECS_REGISTRY_H +#define BLOSC_BLOSC2_CODECS_REGISTRY_H + +#include "blosc2.h" + +#ifdef __cplusplus +extern "C" { +#endif + enum { BLOSC_CODEC_NDLZ = 32, BLOSC_CODEC_ZFP_FIXED_ACCURACY = 33, @@ -25,3 +34,9 @@ typedef struct { // Silence unused codec_info typedef warning static codec_info codec_info_defaults BLOSC_ATTRIBUTE_UNUSED = {0}; + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_CODECS_REGISTRY_H */ diff --git a/src/c-blosc2/include/blosc2/filters-registry.h b/src/c-blosc2/include/blosc2/filters-registry.h index fd74e4e4..b39e2aa7 100644 --- a/src/c-blosc2/include/blosc2/filters-registry.h +++ b/src/c-blosc2/include/blosc2/filters-registry.h @@ -8,10 +8,18 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_BLOSC2_FILTERS_REGISTRY_H +#define BLOSC_BLOSC2_FILTERS_REGISTRY_H + +#ifdef __cplusplus +extern "C" { +#endif + enum { BLOSC_FILTER_NDCELL = 32, BLOSC_FILTER_NDMEAN = 33, - BLOSC_FILTER_BYTEDELTA = 34, + BLOSC_FILTER_BYTEDELTA_BUGGY = 34, // buggy version. See #524 + BLOSC_FILTER_BYTEDELTA = 35, // fixed version }; void register_filters(void); @@ -21,3 +29,9 @@ typedef struct { char *forward; char *backward; } filter_info; + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_FILTERS_REGISTRY_H */ diff --git a/src/c-blosc2/include/blosc2/plugins-utils.h b/src/c-blosc2/include/blosc2/plugins-utils.h deleted file mode 100644 index be23f7e9..00000000 --- a/src/c-blosc2/include/blosc2/plugins-utils.h +++ /dev/null @@ -1,100 +0,0 @@ -/********************************************************************* - Blosc - Blocked Shuffling and Compression Library - - Copyright (C) 2021 The Blosc Developers - https://blosc.org - License: BSD 3-Clause (see LICENSE.txt) - - See LICENSE.txt for details about copyright and rights to use. -**********************************************************************/ - - -#include - -#if defined(_WIN32) -#include -#define PATH_MAX MAX_PATH -#define RTLD_LAZY 0x000 -#define popen _popen -#define pclose _pclose - -static struct { - long lasterror; - const char *err_rutin; -} -var = { - 0, - NULL -}; - -void *dlopen (const char *filename, int flags) { - HINSTANCE hInst; - hInst = LoadLibrary(filename); - if (hInst==NULL) { - var.lasterror = GetLastError(); - var.err_rutin = "dlopen"; - } - - return hInst; -} - -void *dlsym(void *handle, const char *name) { - FARPROC fp; - fp = GetProcAddress((HINSTANCE)handle, name); - if (!fp) { - var.lasterror = GetLastError (); - var.err_rutin = "dlsym"; - } - return (void *)(intptr_t)fp; -} - -int dlclose(void *handle) { - bool ok = FreeLibrary((HINSTANCE)handle); - if (!ok) { - var.lasterror = GetLastError(); - var.err_rutin = "dlclose"; - return BLOSC2_ERROR_FAILURE; - } - return BLOSC2_ERROR_SUCCESS; -} - -const char *dlerror (void) { - static char errstr [88]; - if (var.lasterror) { - sprintf (errstr, "%s error #%ld", var.err_rutin, var.lasterror); - return errstr; - } else { - return NULL; - } -} -#else -#include -#endif - - -static inline void* load_lib(char *plugin_name, char *libpath) { - char python_cmd[PATH_MAX] = {0}; - sprintf(python_cmd, "python -c \"import blosc2_%s; blosc2_%s.print_libpath()\"", plugin_name, plugin_name); - FILE *fp = popen(python_cmd, "r"); - if (fp == NULL) { - BLOSC_TRACE_ERROR("Could not run python"); - return NULL; - } - if (fgets(libpath, PATH_MAX, fp) == NULL) { - BLOSC_TRACE_ERROR("Could not read python output"); - return NULL; - } - pclose(fp); - if (strlen(libpath) == 0) { - BLOSC_TRACE_ERROR("Could not find plugin libpath"); - return NULL; - } - void* loaded_lib; - BLOSC_TRACE_WARNING("libpath for plugin blosc2_%s: %s\n", plugin_name, libpath); - loaded_lib = dlopen(libpath, RTLD_LAZY); - if (loaded_lib == NULL) { - BLOSC_TRACE_ERROR("Attempt to load plugin in path '%s' failed with error: %s", - libpath, dlerror()); - } - return loaded_lib; -} diff --git a/src/c-blosc2/include/blosc2/tuners-registry.h b/src/c-blosc2/include/blosc2/tuners-registry.h index a6310115..6b8b981f 100644 --- a/src/c-blosc2/include/blosc2/tuners-registry.h +++ b/src/c-blosc2/include/blosc2/tuners-registry.h @@ -8,13 +8,20 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_BLOSC2_TUNERS_UTILS_H +#define BLOSC_BLOSC2_TUNERS_UTILS_H + +#ifdef __cplusplus +extern "C" { +#endif + enum { BLOSC_BTUNE = 32, }; void register_tuners(void); -// For dynamically loaded tunes +// For dynamically loaded tuners typedef struct { char *init; char *next_blocksize; @@ -22,3 +29,9 @@ typedef struct { char *update; char *free; } tuner_info; + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_TUNERS_UTILS_H */ diff --git a/src/c-blosc2/plugins/CMakeLists.txt b/src/c-blosc2/plugins/CMakeLists.txt index 1b46ac22..d0064ceb 100644 --- a/src/c-blosc2/plugins/CMakeLists.txt +++ b/src/c-blosc2/plugins/CMakeLists.txt @@ -7,7 +7,7 @@ # See LICENSE.txt for details about copyright and rights to use. add_subdirectory(codecs) -add_subdirectory(tunes) +add_subdirectory(tuners) add_subdirectory(filters) set(SOURCES ${SOURCES} ${PROJECT_SOURCE_DIR}/plugins/plugin_utils.c PARENT_SCOPE) diff --git a/src/c-blosc2/plugins/README.md b/src/c-blosc2/plugins/README.md index c2751b4b..095f922b 100644 --- a/src/c-blosc2/plugins/README.md +++ b/src/c-blosc2/plugins/README.md @@ -1,42 +1,22 @@ Plugins registry for Blosc users ================================ -Blosc has a tradition of supporting different filters and codecs for compressing data, -and it was up to the user to choose one or another depending on his needs. -However, it is clear that there will always be scenarios where a richer variety -of them could be useful. So the Blosc team has set new goals: +Blosc has a tradition of supporting different filters and codecs for compressing data, and it has been up to the user to choose one or another depending on their needs. However, it is clear that there are always scenarios where a richer variety of them could be useful. Therefore, the Blosc team has set new goals: -1) Implement a way for users to locally register filters and codecs so that they can use - them in their setup at will. +1) Implement a way for users to register filters and codecs locally so that they can use them in their setup as needed. -2) Set up a central registry so that *other* users can make use of these filters and codecs - without interfering with other ones that have been created by other users. - -As a bonus, those codecs and filters accepted in the central registry and meeting the quality standards -defined in these guidelines will be distributed *inside* the C-Blosc2 library, -allowing a much easier way for others to use them: install C-Blosc2 library and you are all set. -Of course, to achieve such a status, plugins will require a careful testing process described below. +2) Set up a central registry so that other users can use these filters and codecs without interfering with filters and codecs created by other users. +As a bonus, codecs and filters that are accepted into the central registry and meet the quality standards defined in these guidelines will be distributed *within* the C-Blosc2 library. This allows for a much easier way for others to use them; simply install the C-Blosc2 library and you're all set. Of course, to achieve such a status, plugins will require a careful testing process as described below. Plugin types -------------- The plugins that are registered in the repository can be codecs or filters. -A **codec** is a program able to compress and decompress a digital data stream -with the objective of reduce dataset size to enable a faster transmission -of data. -Some codecs used by Blosc are e.g. *BLOSCLZ*, *LZ4* and *ZSTANDARD*. - -A **filter** is a program that reorders the data without -changing its size, so that the initial and final size are equal. -A filter consists of encoder and decoder. Filter encoder is applied before -using the codec compressor (or codec encoder) in order to make data easier to compress -and filter decoder is used after codec decompressor (or codec decoder) to recover -the original data arrangement. -Some filters actually used by Blosc are e.g. *SHUFFLE*, which rearranges data -based on the typesize, or *TRUNC*, which zeroes mantissa bits to reduce -the precision of (floating point) data, and hence, increase the compression ratio. +A codec is a program that compresses and decompresses digital data streams with the objective of reducing dataset size to enable faster transmission of data. Blosc uses various codecs, including BLOSCLZ, LZ4, and ZSTANDARD. + +A filter is a program that rearranges data without changing its size, so that the initial and final sizes remain equal. A filter consists of an encoder and decoder. The filter encoder is applied before using the codec compressor (or codec encoder) to make the data easier to compress, while the filter decoder is used after the codec decompressor (or codec decoder) to restore the original data arrangement. Some filters used by Blosc include SHUFFLE, which rearranges data based on the type size, and TRUNC, which zeroes the mantissa bits to reduce the precision of (floating point) data and increase the compression ratio. Here it is an example on how the compression process goes: @@ -51,8 +31,7 @@ And the decompression process: | c_src | -----------> | tmp | ----------> | src | -------- ------------------- ------------------- -Moreover, during the pipeline process you can use even 6 different -filters ordered as you prefer. +Furthermore, during the pipeline process, you can use up to six different filters, ordered in any way you prefer. Blosc global registered plugins vs user registered plugins @@ -111,15 +90,14 @@ Steps * The advantages and disadvantages of the plugin compared to the rest. -4. To register a plugin the user must choose a plugin ID between *BLOSC2_GLOBAL_REGISTERED_FILTERS_START* and *BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP* and - write it at `include/blosc2/codecs-registry.h` - or `include/blosc2/filters-registry.h` depending on the plugin type. Then, you have to edit `include/blosc2/codecs-registry.c`or +4. To register a plugin the user must choose a plugin ID between *BLOSC2_GLOBAL_REGISTERED_FILTERS_START* + and *BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP* and write it at `include/blosc2/codecs-registry.h` or + `include/blosc2/filters-registry.h` depending on the plugin type. - `include/blosc2/filters-registry.c` in the next way: + Then, you have to edit `plugins/codecs/codecs-registry.c`or `plugins/codecs/filters-registry.c` in the next way: - At the top it must be added `#include "plugin_folder/plugin_header.h"`, - - and into the register function you must follow the same steps that were done for the existing plugins. + At the top it must be added `#include "plugin_folder/plugin_header.h"`, and into the register function you must + follow the same steps that were done for the existing plugins. 5. Finally, the Blosc development team will carry out the evaluation process to decide whether the plugin is useful and hence, candidate to be integrated into the C-Blosc2 diff --git a/src/c-blosc2/plugins/codecs/codecs-registry.c b/src/c-blosc2/plugins/codecs/codecs-registry.c index db5c9f02..def6452d 100644 --- a/src/c-blosc2/plugins/codecs/codecs-registry.c +++ b/src/c-blosc2/plugins/codecs/codecs-registry.c @@ -4,10 +4,11 @@ License: BSD 3-Clause (see LICENSE.txt) */ -#include "blosc-private.h" #include "blosc2/codecs-registry.h" #include "ndlz/ndlz.h" #include "zfp/blosc2-zfp.h" +#include "blosc-private.h" +#include "blosc2.h" void register_codecs(void) { @@ -15,8 +16,8 @@ void register_codecs(void) { ndlz.compcode = BLOSC_CODEC_NDLZ; ndlz.version = 1; ndlz.complib = BLOSC_CODEC_NDLZ; - ndlz.encoder = ndlz_compress; - ndlz.decoder = ndlz_decompress; + ndlz.encoder = &ndlz_compress; + ndlz.decoder = &ndlz_decompress; ndlz.compname = "ndlz"; register_codec_private(&ndlz); @@ -24,8 +25,8 @@ void register_codecs(void) { zfp_acc.compcode = BLOSC_CODEC_ZFP_FIXED_ACCURACY; zfp_acc.version = 1; zfp_acc.complib = BLOSC_CODEC_ZFP_FIXED_ACCURACY; - zfp_acc.encoder = zfp_acc_compress; - zfp_acc.decoder = zfp_acc_decompress; + zfp_acc.encoder = &zfp_acc_compress; + zfp_acc.decoder = &zfp_acc_decompress; zfp_acc.compname = "zfp_acc"; register_codec_private(&zfp_acc); @@ -33,8 +34,8 @@ void register_codecs(void) { zfp_prec.compcode = BLOSC_CODEC_ZFP_FIXED_PRECISION; zfp_prec.version = 1; zfp_prec.complib = BLOSC_CODEC_ZFP_FIXED_PRECISION; - zfp_prec.encoder = zfp_prec_compress; - zfp_prec.decoder = zfp_prec_decompress; + zfp_prec.encoder = &zfp_prec_compress; + zfp_prec.decoder = &zfp_prec_decompress; zfp_prec.compname = "zfp_prec"; register_codec_private(&zfp_prec); @@ -42,8 +43,8 @@ void register_codecs(void) { zfp_rate.compcode = BLOSC_CODEC_ZFP_FIXED_RATE; zfp_rate.version = 1; zfp_rate.complib = BLOSC_CODEC_ZFP_FIXED_RATE; - zfp_rate.encoder = zfp_rate_compress; - zfp_rate.decoder = zfp_rate_decompress; + zfp_rate.encoder = &zfp_rate_compress; + zfp_rate.decoder = &zfp_rate_decompress; zfp_rate.compname = "zfp_rate"; register_codec_private(&zfp_rate); } diff --git a/src/c-blosc2/plugins/codecs/ndlz/CMakeLists.txt b/src/c-blosc2/plugins/codecs/ndlz/CMakeLists.txt index 24424fa1..5e06cfbb 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/CMakeLists.txt +++ b/src/c-blosc2/plugins/codecs/ndlz/CMakeLists.txt @@ -19,15 +19,13 @@ if(BUILD_TESTS) add_executable(test_ndlz test_ndlz.c) # Define the BLOSC_TESTING symbol so normally-hidden functions # aren't hidden from the view of the test programs. - set_property( - TARGET test_ndlz - APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + target_compile_definitions(test_ndlz PUBLIC BLOSC_TESTING) - target_link_libraries(test_ndlz blosc_testing) + target_link_libraries(test_ndlz PUBLIC blosc_testing) # tests - add_test(NAME test_plugin_test_ndlz - COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + add_test(NAME test_plugin_test_ndlz + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) # Copy test files file(GLOB TESTS_DATA ../../test_data/example_s*.b2nd) diff --git a/src/c-blosc2/plugins/codecs/ndlz/ndlz-private.h b/src/c-blosc2/plugins/codecs/ndlz/ndlz-private.h index c73239a0..694713ce 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/ndlz-private.h +++ b/src/c-blosc2/plugins/codecs/ndlz/ndlz-private.h @@ -8,28 +8,18 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ - - #ifndef NDLZ_PRIVATE_H #define NDLZ_PRIVATE_H -#include "context.h" +#include -#if defined (__cplusplus) -extern "C" { -#endif #define XXH_INLINE_ALL -#define NDLZ_ERROR_NULL(pointer) \ - do { \ - if ((pointer) == NULL) { \ - return 0; \ - } \ +#define NDLZ_ERROR_NULL(pointer) \ + do { \ + if ((pointer) == NULL) { \ + return 0; \ + } \ } while (0) - -#if defined (__cplusplus) -} -#endif - #endif /* NDLZ_PRIVATE_H */ diff --git a/src/c-blosc2/plugins/codecs/ndlz/ndlz.c b/src/c-blosc2/plugins/codecs/ndlz/ndlz.c index 128be185..07fb73ae 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/ndlz.c +++ b/src/c-blosc2/plugins/codecs/ndlz/ndlz.c @@ -15,12 +15,10 @@ linear one. **********************************************************************/ - -#include -#include "ndlz.h" #include "ndlz-private.h" #include "ndlz4x4.h" #include "ndlz8x8.h" +#include "ndlz.h" int ndlz_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_cparams *cparams, const void *chunk) { diff --git a/src/c-blosc2/plugins/codecs/ndlz/ndlz.h b/src/c-blosc2/plugins/codecs/ndlz/ndlz.h index ed3efbdb..ac9fdcb2 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/ndlz.h +++ b/src/c-blosc2/plugins/codecs/ndlz/ndlz.h @@ -8,15 +8,12 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_PLUGINS_CODECS_NDLZ_NDLZ_H +#define BLOSC_PLUGINS_CODECS_NDLZ_NDLZ_H +#include "blosc2.h" -#ifndef NDLZ_H -#define NDLZ_H -#include "context.h" - -#if defined (__cplusplus) -extern "C" { -#endif +#include int ndlz_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_cparams *cparams, const void* chunk); @@ -24,8 +21,4 @@ int ndlz_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int3 int ndlz_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_dparams *dparams, const void* chunk); -#if defined (__cplusplus) -} -#endif - -#endif /* NDLZ_H */ +#endif /* BLOSC_PLUGINS_CODECS_NDLZ_NDLZ_H */ diff --git a/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.c b/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.c index 062472e3..60e0813b 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.c +++ b/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.c @@ -15,13 +15,13 @@ linear one. **********************************************************************/ - -#include #include "ndlz4x4.h" #include "ndlz.h" #include "xxhash.h" #include "../plugins/plugin_utils.h" +#include +#include /* * Give hints to the compiler for branch prediction optimization. @@ -59,6 +59,8 @@ int ndlz4_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_cparams *cparams) { BLOSC_UNUSED_PARAM(meta); + BLOSC_ERROR_NULL(cparams, BLOSC2_ERROR_NULL_POINTER); + BLOSC_ERROR_NULL(cparams->schunk, BLOSC2_ERROR_NULL_POINTER); uint8_t *smeta; int32_t smeta_len; @@ -513,6 +515,8 @@ int ndlz4_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, i uint8_t meta, blosc2_dparams *dparams) { BLOSC_UNUSED_PARAM(meta); BLOSC_UNUSED_PARAM(dparams); + BLOSC_ERROR_NULL(input, BLOSC2_ERROR_NULL_POINTER); + BLOSC_ERROR_NULL(output, BLOSC2_ERROR_NULL_POINTER); uint8_t *ip = (uint8_t *) input; uint8_t *ip_limit = ip + input_len; @@ -541,7 +545,7 @@ int ndlz4_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, i eshape[0] = ((blockshape[0] + 3) / 4) * 4; eshape[1] = ((blockshape[1] + 3) / 4) * 4; - if (NDLZ_UNEXPECT_CONDITIONAL(output_len < (int32_t) (blockshape[0] * blockshape[1]))) { + if (NDLZ_UNEXPECT_CONDITIONAL((int64_t)output_len < (int64_t)blockshape[0] * (int64_t)blockshape[1])) { return 0; } memset(op, 0, blockshape[0] * blockshape[1]); diff --git a/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.h b/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.h index cefd85c8..23a5c0e1 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.h +++ b/src/c-blosc2/plugins/codecs/ndlz/ndlz4x4.h @@ -8,25 +8,19 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_PLUGINS_CODECS_NDLZ_NDLZ4X4_H +#define BLOSC_PLUGINS_CODECS_NDLZ_NDLZ4X4_H - -#ifndef NDLZ4_H -#define NDLZ4_H -#include "context.h" - -#if defined (__cplusplus) -extern "C" { -#endif -#include "ndlz.h" #include "ndlz-private.h" +#include "ndlz.h" +#include "blosc2.h" + /* #include #include "blosc2/blosc2-common.h" #include "fastcopy.h" */ - - /** Compress a block of data in the input buffer and returns the size of compressed block. The size of input buffer is specified by @@ -65,8 +59,4 @@ int ndlz4_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int int ndlz4_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_dparams *dparams); -#if defined (__cplusplus) -} -#endif - -#endif /* NDLZ4_H */ +#endif /* BLOSC_PLUGINS_CODECS_NDLZ_NDLZ4X4_H */ diff --git a/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.c b/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.c index 6949184d..59f63ab7 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.c +++ b/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.c @@ -15,13 +15,13 @@ linear one. **********************************************************************/ - -#include #include "ndlz8x8.h" #include "ndlz.h" #include "xxhash.h" #include "../plugins/plugin_utils.h" +#include +#include /* * Give hints to the compiler for branch prediction optimization. @@ -59,6 +59,8 @@ int ndlz8_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_cparams *cparams) { BLOSC_UNUSED_PARAM(meta); + BLOSC_ERROR_NULL(cparams, BLOSC2_ERROR_NULL_POINTER); + BLOSC_ERROR_NULL(cparams->schunk, BLOSC2_ERROR_NULL_POINTER); uint8_t *smeta; int32_t smeta_len; @@ -430,6 +432,8 @@ int ndlz8_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, i uint8_t meta, blosc2_dparams *dparams) { BLOSC_UNUSED_PARAM(meta); BLOSC_UNUSED_PARAM(dparams); + BLOSC_ERROR_NULL(input, BLOSC2_ERROR_NULL_POINTER); + BLOSC_ERROR_NULL(output, BLOSC2_ERROR_NULL_POINTER); const int cell_shape = 8; const int cell_size = 64; @@ -458,8 +462,7 @@ int ndlz8_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, i ip += 4; eshape[0] = ((blockshape[0] + 7) / cell_shape) * cell_shape; eshape[1] = ((blockshape[1] + 7) / cell_shape) * cell_shape; - - if (NDLZ_UNEXPECT_CONDITIONAL(output_len < blockshape[0] * blockshape[1])) { + if (NDLZ_UNEXPECT_CONDITIONAL((int64_t)output_len < (int64_t)blockshape[0] * (int64_t)blockshape[1])) { return 0; } memset(op, 0, blockshape[0] * blockshape[1]); diff --git a/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.h b/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.h index d3fc107b..4674a7c8 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.h +++ b/src/c-blosc2/plugins/codecs/ndlz/ndlz8x8.h @@ -8,17 +8,12 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef NDLZ8_H -#define NDLZ8_H +#ifndef BLOSC_PLUGINS_CODECS_NDLZ_NDLZ8X8_H +#define BLOSC_PLUGINS_CODECS_NDLZ_NDLZ8X8_H -#include "context.h" - -#if defined (__cplusplus) -extern "C" { -#endif - -#include "ndlz.h" #include "ndlz-private.h" +#include "ndlz.h" +#include "blosc2.h" /** Compress a block of data in the input buffer and returns the size of @@ -58,8 +53,4 @@ int ndlz8_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int int ndlz8_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_dparams *dparams); -#if defined (__cplusplus) -} -#endif - -#endif /* NDLZ8_H */ +#endif /* BLOSC_PLUGINS_CODECS_NDLZ_NDLZ8X8_H */ diff --git a/src/c-blosc2/plugins/codecs/ndlz/test_ndlz.c b/src/c-blosc2/plugins/codecs/ndlz/test_ndlz.c index efff0194..64e378d1 100644 --- a/src/c-blosc2/plugins/codecs/ndlz/test_ndlz.c +++ b/src/c-blosc2/plugins/codecs/ndlz/test_ndlz.c @@ -30,11 +30,12 @@ **********************************************************************/ -#include -#include "blosc2.h" +#include "b2nd.h" #include "blosc2/codecs-registry.h" +#include "blosc2.h" + #include -#include "b2nd.h" +#include static int test_ndlz_4(blosc2_schunk *schunk) { diff --git a/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.c b/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.c index 51a220e0..41d08d0d 100644 --- a/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.c +++ b/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.c @@ -4,16 +4,20 @@ License: BSD 3-Clause (see LICENSE.txt) */ -#include "blosc2.h" #include "blosc-private.h" -#include "frame.h" -#include "blosc2/codecs-registry.h" #include "zfp.h" #include "blosc2-zfp.h" -#include -#include "context.h" -#include "assert.h" +#include "../plugins/codecs/zfp/zfp-private.h" #include "../plugins/plugin_utils.h" +#include "context.h" +#include "frame.h" +#include "blosc2/codecs-registry.h" +#include "blosc2.h" + +#include +#include +#include +#include int zfp_acc_compress(const uint8_t *input, int32_t input_len, uint8_t *output, @@ -22,6 +26,7 @@ int zfp_acc_compress(const uint8_t *input, int32_t input_len, uint8_t *output, ZFP_ERROR_NULL(input); ZFP_ERROR_NULL(output); ZFP_ERROR_NULL(cparams); + ZFP_ERROR_NULL(cparams->schunk); double tol = (int8_t) meta; int8_t ndim; @@ -142,6 +147,7 @@ int zfp_acc_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, ZFP_ERROR_NULL(input); ZFP_ERROR_NULL(output); ZFP_ERROR_NULL(dparams); + ZFP_ERROR_NULL(dparams->schunk); BLOSC_UNUSED_PARAM(chunk); blosc2_schunk *sc = dparams->schunk; @@ -237,6 +243,7 @@ int zfp_prec_compress(const uint8_t *input, int32_t input_len, uint8_t *output, ZFP_ERROR_NULL(input); ZFP_ERROR_NULL(output); ZFP_ERROR_NULL(cparams); + ZFP_ERROR_NULL(cparams->schunk); int8_t ndim; int64_t *shape = malloc(8 * sizeof(int64_t)); @@ -381,6 +388,7 @@ int zfp_prec_decompress(const uint8_t *input, int32_t input_len, uint8_t *output ZFP_ERROR_NULL(input); ZFP_ERROR_NULL(output); ZFP_ERROR_NULL(dparams); + ZFP_ERROR_NULL(dparams->schunk); BLOSC_UNUSED_PARAM(chunk); blosc2_schunk *sc = dparams->schunk; @@ -500,6 +508,7 @@ int zfp_rate_compress(const uint8_t *input, int32_t input_len, uint8_t *output, ZFP_ERROR_NULL(input); ZFP_ERROR_NULL(output); ZFP_ERROR_NULL(cparams); + ZFP_ERROR_NULL(cparams->schunk); double ratio = (double) meta / 100.0; int8_t ndim; @@ -631,6 +640,7 @@ int zfp_rate_decompress(const uint8_t *input, int32_t input_len, uint8_t *output ZFP_ERROR_NULL(input); ZFP_ERROR_NULL(output); ZFP_ERROR_NULL(dparams); + ZFP_ERROR_NULL(dparams->schunk); BLOSC_UNUSED_PARAM(chunk); blosc2_schunk *sc = dparams->schunk; diff --git a/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.h b/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.h index 20684a62..51ba8f63 100644 --- a/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.h +++ b/src/c-blosc2/plugins/codecs/zfp/blosc2-zfp.h @@ -8,18 +8,12 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_PLUGINS_CODECS_ZFP_BLOSC2_ZFP_H +#define BLOSC_PLUGINS_CODECS_ZFP_BLOSC2_ZFP_H +#include "blosc2.h" -#ifndef BLOSC2_ZFP_H -#define BLOSC2_ZFP_H - -#include "zfp-private.h" -#include "../plugins/plugin_utils.h" - -#if defined (__cplusplus) -extern "C" { -#endif - +#include int zfp_acc_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_cparams *cparams, const void *chunk); @@ -41,9 +35,4 @@ int zfp_rate_decompress(const uint8_t *input, int32_t input_len, uint8_t *output int zfp_getcell(void *thread_context, const uint8_t *block, int32_t cbytes, uint8_t *dest, int32_t destsize); - -#if defined (__cplusplus) -} -#endif - -#endif /* BLOSC2_ZFP_H */ +#endif /* BLOSC_PLUGINS_CODECS_ZFP_BLOSC2_ZFP_H */ diff --git a/src/c-blosc2/plugins/codecs/zfp/test_zfp_acc_float.c b/src/c-blosc2/plugins/codecs/zfp/test_zfp_acc_float.c index 88029390..0ade56bb 100644 --- a/src/c-blosc2/plugins/codecs/zfp/test_zfp_acc_float.c +++ b/src/c-blosc2/plugins/codecs/zfp/test_zfp_acc_float.c @@ -15,13 +15,14 @@ **********************************************************************/ -#include -#include -#include "blosc2.h" -#include "blosc2/codecs-registry.h" #include "blosc-private.h" -#include #include "b2nd.h" +#include "blosc2/codecs-registry.h" +#include "blosc2.h" + +#include +#include +#include static int test_zfp_acc_float(blosc2_schunk *schunk) { diff --git a/src/c-blosc2/plugins/codecs/zfp/test_zfp_prec_float.c b/src/c-blosc2/plugins/codecs/zfp/test_zfp_prec_float.c index 83f21e31..22e4ce6a 100644 --- a/src/c-blosc2/plugins/codecs/zfp/test_zfp_prec_float.c +++ b/src/c-blosc2/plugins/codecs/zfp/test_zfp_prec_float.c @@ -15,13 +15,14 @@ **********************************************************************/ -#include -#include -#include "blosc2.h" -#include "blosc2/codecs-registry.h" #include "blosc-private.h" -#include #include "b2nd.h" +#include "blosc2/codecs-registry.h" +#include "blosc2.h" + +#include +#include +#include static int test_zfp_prec_float(blosc2_schunk *schunk) { diff --git a/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_float.c b/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_float.c index 57b9f0d4..de99cc76 100644 --- a/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_float.c +++ b/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_float.c @@ -15,13 +15,14 @@ **********************************************************************/ -#include -#include "blosc2.h" -#include "blosc2/codecs-registry.h" #include "blosc-private.h" -#include #include "b2nd.h" +#include "blosc2/codecs-registry.h" +#include "blosc2.h" + +#include #include +#include static int test_zfp_rate_float(blosc2_schunk *schunk, double tolerance) { diff --git a/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_getitem.c b/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_getitem.c index 65a2bdc0..a628d31a 100644 --- a/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_getitem.c +++ b/src/c-blosc2/plugins/codecs/zfp/test_zfp_rate_getitem.c @@ -15,13 +15,14 @@ **********************************************************************/ -#include -#include -#include -#include "blosc2.h" -#include "blosc2/codecs-registry.h" #include "blosc-private.h" #include "b2nd.h" +#include "blosc2/codecs-registry.h" +#include "blosc2.h" + +#include +#include +#include static int test_zfp_rate_getitem_float(blosc2_schunk *schunk) { diff --git a/src/c-blosc2/plugins/codecs/zfp/zfp-private.h b/src/c-blosc2/plugins/codecs/zfp/zfp-private.h index d56f33e7..a5a62328 100644 --- a/src/c-blosc2/plugins/codecs/zfp/zfp-private.h +++ b/src/c-blosc2/plugins/codecs/zfp/zfp-private.h @@ -8,30 +8,21 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#ifndef BLOSC_PLUGINS_CODECS_ZFP_ZFP_PRIVATE_H +#define BLOSC_PLUGINS_CODECS_ZFP_ZFP_PRIVATE_H - -#ifndef ZFP_PRIVATE_H -#define ZFP_PRIVATE_H +#include #define ZFP_MAX_DIM 4 #define ZFP_CELL_SHAPE 4 - -#if defined (__cplusplus) -extern "C" { -#endif #define XXH_INLINE_ALL -#define ZFP_ERROR_NULL(pointer) \ - do { \ - if ((pointer) == NULL) { \ - return 0; \ - } \ +#define ZFP_ERROR_NULL(pointer) \ + do { \ + if ((pointer) == NULL) { \ + return 0; \ + } \ } while (0) - -#if defined (__cplusplus) -} -#endif - -#endif /* ZFP_PRIVATE_H */ +#endif /* BLOSC_PLUGINS_CODECS_ZFP_ZFP_PRIVATE_H */ diff --git a/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c b/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c index 6506b607..32e81f7f 100644 --- a/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c +++ b/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c @@ -12,11 +12,13 @@ // https://aras-p.info/blog/2023/03/01/Float-Compression-7-More-Filtering-Optimization/ // This requires Intel SSE4.1 and ARM64 NEON, which should be widely available by now. -#include #include "bytedelta.h" -#include #include "../plugins/plugin_utils.h" -#include "../include/blosc2/filters-registry.h" +#include "blosc2/filters-registry.h" +#include "blosc2.h" + +#include +#include #if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64 // SSSE3 code path for x64/x64 @@ -44,6 +46,8 @@ bytes16 simd_prefix_sum(bytes16 x) return x; } +uint8_t simd_get_last(bytes16 x) { return (_mm_extract_epi16(x, 7) >> 8) & 0xFF; } + #elif defined(__aarch64__) || defined(_M_ARM64) // ARM v8 NEON code path #define CPU_HAS_SIMD 1 @@ -70,6 +74,8 @@ bytes16 simd_prefix_sum(bytes16 x) return x; } +uint8_t simd_get_last(bytes16 x) { return vgetq_lane_u8(x, 15); } + #endif @@ -91,6 +97,7 @@ int bytedelta_forward(const uint8_t *input, uint8_t *output, int32_t length, uin const int stream_len = length / typesize; for (int ich = 0; ich < typesize; ++ich) { int ip = 0; + uint8_t _v2 = 0; // SIMD delta within each channel, store #if defined(CPU_HAS_SIMD) bytes16 v2 = {0}; @@ -102,9 +109,11 @@ int bytedelta_forward(const uint8_t *input, uint8_t *output, int32_t length, uin output += 16; v2 = v; } + if (stream_len > 15) { + _v2 = simd_get_last(v2); + } #endif // #if defined(CPU_HAS_SIMD) // scalar leftover - uint8_t _v2 = 0; for (; ip < stream_len ; ip++) { uint8_t v = *input; input++; @@ -119,7 +128,100 @@ int bytedelta_forward(const uint8_t *input, uint8_t *output, int32_t length, uin // Fetch 16b from N streams, sum SIMD undelta int bytedelta_backward(const uint8_t *input, uint8_t *output, int32_t length, uint8_t meta, - blosc2_dparams *dparams, uint8_t id) { + blosc2_dparams *dparams, uint8_t id) { + BLOSC_UNUSED_PARAM(id); + + int typesize = meta; + if (typesize == 0) { + if (dparams->schunk == NULL) { + BLOSC_TRACE_ERROR("When meta is 0, you need to be on a schunk!"); + BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + } + blosc2_schunk* schunk = (blosc2_schunk*)(dparams->schunk); + typesize = schunk->typesize; + } + + const int stream_len = length / typesize; + for (int ich = 0; ich < typesize; ++ich) { + int ip = 0; + uint8_t _v2 = 0; + // SIMD fetch 16 bytes from each channel, prefix-sum un-delta +#if defined(CPU_HAS_SIMD) + bytes16 v2 = {0}; + for (; ip < stream_len - 15; ip += 16) { + bytes16 v = simd_load(input); + input += 16; + // un-delta via prefix sum + v2 = simd_add(simd_prefix_sum(v), simd_duplane15(v2)); + simd_store(output, v2); + output += 16; + } + if (stream_len > 15) { + _v2 = simd_get_last(v2); + } +#endif // #if defined(CPU_HAS_SIMD) + // scalar leftover + for (; ip < stream_len; ip++) { + uint8_t v = *input + _v2; + input++; + *output = v; + output++; + _v2 = v; + } + } + + return BLOSC2_ERROR_SUCCESS; +} + +// This is the original (and buggy) version of bytedelta. It is kept here for backwards compatibility. +// See #524 for details. +// Fetch 16b from N streams, compute SIMD delta +int bytedelta_forward_buggy(const uint8_t *input, uint8_t *output, int32_t length, + uint8_t meta, blosc2_cparams *cparams, uint8_t id) { + BLOSC_UNUSED_PARAM(id); + + int typesize = meta; + if (typesize == 0) { + if (cparams->schunk == NULL) { + BLOSC_TRACE_ERROR("When meta is 0, you need to be on a schunk!"); + BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + } + blosc2_schunk* schunk = (blosc2_schunk*)(cparams->schunk); + typesize = schunk->typesize; + } + + const int stream_len = length / typesize; + for (int ich = 0; ich < typesize; ++ich) { + int ip = 0; + // SIMD delta within each channel, store +#if defined(CPU_HAS_SIMD) + bytes16 v2 = {0}; + for (; ip < stream_len - 15; ip += 16) { + bytes16 v = simd_load(input); + input += 16; + bytes16 delta = simd_sub(v, simd_concat(v, v2)); + simd_store(output, delta); + output += 16; + v2 = v; + } +#endif // #if defined(CPU_HAS_SIMD) + // scalar leftover + uint8_t _v2 = 0; + for (; ip < stream_len ; ip++) { + uint8_t v = *input; + input++; + *output = v - _v2; + output++; + _v2 = v; + } + } + + return BLOSC2_ERROR_SUCCESS; +} + +// Fetch 16b from N streams, sum SIMD undelta +int bytedelta_backward_buggy(const uint8_t *input, uint8_t *output, int32_t length, + uint8_t meta, blosc2_dparams *dparams, uint8_t id) { BLOSC_UNUSED_PARAM(id); int typesize = meta; diff --git a/src/c-blosc2/plugins/filters/bytedelta/bytedelta.h b/src/c-blosc2/plugins/filters/bytedelta/bytedelta.h index 99e5e5ce..48c1f393 100644 --- a/src/c-blosc2/plugins/filters/bytedelta/bytedelta.h +++ b/src/c-blosc2/plugins/filters/bytedelta/bytedelta.h @@ -8,12 +8,23 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#pragma once +#ifndef BLOSC_PLUGINS_FILTERS_BYTEDELTA_BYTEDELTA_H +#define BLOSC_PLUGINS_FILTERS_BYTEDELTA_BYTEDELTA_H + +#include "blosc2.h" + #include -#include int bytedelta_forward(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_cparams* cparams, uint8_t id); int bytedelta_backward(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, - blosc2_dparams* dparams, uint8_t id); + blosc2_dparams* dparams, uint8_t id); + +int bytedelta_forward_buggy(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, + blosc2_cparams* cparams, uint8_t id); + +int bytedelta_backward_buggy(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, + blosc2_dparams* dparams, uint8_t id); + +#endif /* BLOSC_PLUGINS_FILTERS_BYTEDELTA_BYTEDELTA_H */ diff --git a/src/c-blosc2/plugins/filters/bytedelta/test_bytedelta.c b/src/c-blosc2/plugins/filters/bytedelta/test_bytedelta.c index 20079b2e..fd27ca8b 100644 --- a/src/c-blosc2/plugins/filters/bytedelta/test_bytedelta.c +++ b/src/c-blosc2/plugins/filters/bytedelta/test_bytedelta.c @@ -15,25 +15,101 @@ To run: $ ./test_bytedelta + Testing bytedelta with write_correct=1, read_correct=1 + Testing bytedelta with write_correct=0, read_correct=1 + Testing bytedelta with write_correct=1, read_correct=0 + Testing bytedelta with write_correct=0, read_correct=0 Successful roundtrip! - Compression: 41472 -> 4937 (8.4x) - rand: 36535 obtained + Compression: 41472 -> 5599 (7.4x) + rand: saved 35873 bytes + Testing bytedelta with write_correct=1, read_correct=1 + Testing bytedelta with write_correct=0, read_correct=1 + Testing bytedelta with write_correct=1, read_correct=0 + Testing bytedelta with write_correct=0, read_correct=0 Successful roundtrip! - Compression: 1792 -> 1005 (1.8x) - mixed_values: 787 obtained + Compression: 1792 -> 499 (3.6x) + mixed_values: saved 1293 bytes + Testing bytedelta with write_correct=1, read_correct=1 + Testing bytedelta with write_correct=0, read_correct=1 + Testing bytedelta with write_correct=1, read_correct=0 + Testing bytedelta with write_correct=0, read_correct=0 Successful roundtrip! - Compression: 16128 -> 1599 (10.1x) - arange_like: 14529 obtained + Compression: 16128 -> 1473 (10.9x) + arange_like: saved 14655 bytes **********************************************************************/ -#include -#include #include "blosc2/filters-registry.h" #include "b2nd.h" +#include +#include +#include +#include + +/* The original implementation of the bytedelta filter had incorrect + * roundtrip behavior between SIMD and non-SIMD binaries. This filter provides + * the correct behavior regardless of SIMD availability. + * See https://github.com/Blosc/c-blosc2/issues/524 */ +int correct_bytedelta_forward(const uint8_t *input, uint8_t *output, int32_t length, uint8_t meta, + blosc2_cparams *cparams, uint8_t id) +{ + BLOSC_UNUSED_PARAM(id); + int typesize = meta; + if (typesize == 0) { + if (cparams->schunk == NULL) { + BLOSC_TRACE_ERROR("When meta is 0, you need to be on a schunk!"); + BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + } + blosc2_schunk* schunk = (blosc2_schunk*)(cparams->schunk); + typesize = schunk->typesize; + } + int stream_len = length / typesize; + for (int ich = 0; ich < typesize; ++ich) { + int ip = 0; + uint8_t _v2 = 0; + for (; ip < stream_len ; ip++) { + uint8_t v = *input; + input++; + *output = v - _v2; + output++; + _v2 = v; + } + } + return BLOSC2_ERROR_SUCCESS; +} + +int correct_bytedelta_backward(const uint8_t *input, uint8_t *output, int32_t length, uint8_t meta, + blosc2_dparams *dparams, uint8_t id) { + BLOSC_UNUSED_PARAM(id); + int typesize = meta; + if (typesize == 0) { + if (dparams->schunk == NULL) { + BLOSC_TRACE_ERROR("When meta is 0, you need to be on a schunk!"); + BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + } + blosc2_schunk* schunk = (blosc2_schunk*)(dparams->schunk); + typesize = schunk->typesize; + } + + const int stream_len = length / typesize; + for (int ich = 0; ich < typesize; ++ich) { + int ip = 0; + uint8_t _v2 = 0; + for (; ip < stream_len; ip++) { + uint8_t v = *input + _v2; + input++; + *output = v; + output++; + _v2 = v; + } + } + + return BLOSC2_ERROR_SUCCESS; +} + static int test_bytedelta(blosc2_schunk *schunk) { int64_t nchunks = schunk->nchunks; @@ -46,75 +122,103 @@ static int test_bytedelta(blosc2_schunk *schunk) { uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); uint8_t *data_dest = malloc(chunksize); - /* Create a context for compression */ - blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; - cparams.typesize = schunk->typesize; - cparams.compcode = BLOSC_LZ4; - cparams.filters[BLOSC2_MAX_FILTERS - 2] = BLOSC_SHUFFLE; - cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_FILTER_BYTEDELTA; - cparams.filters_meta[BLOSC2_MAX_FILTERS - 1] = 0; // 0 means typesize when using schunks - cparams.clevel = 9; - cparams.nthreads = 1; - cparams.blocksize = schunk->blocksize; - cparams.schunk = schunk; - blosc2_context *cctx; - cctx = blosc2_create_cctx(cparams); - if (cctx == NULL) { - printf("Cannot create compression context!\n"); - return -1; - } - - blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; - dparams.nthreads = 1; - dparams.schunk = schunk; - blosc2_context *dctx; - dctx = blosc2_create_dctx(dparams); - if (cctx == NULL) { - printf("Cannot create decompression context!\n"); + blosc2_filter correct_bytedelta = {.id = 250, .name = "bytedelta_correct", .version = 1, + .forward = correct_bytedelta_forward, .backward = correct_bytedelta_backward}; + if (blosc2_register_filter(&correct_bytedelta)) { + printf("Cannot register bytedelta filter!"); return -1; } - for (int ci = 0; ci < nchunks; ci++) { - decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); - if (decompressed < 0) { - printf("Error decompressing chunk \n"); + // Test each pair of possible forward/backward implementations: + for (int direction = 0; direction < 4; direction++) { + bool write_correct = (direction % 2 == 0); + bool read_correct = (direction / 2 == 0); + printf("Testing bytedelta with write_correct=%d, read_correct=%d\n", write_correct, read_correct); + + /* Create a context for compression */ + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_LZ4; + cparams.filters[BLOSC2_MAX_FILTERS - 2] = BLOSC_SHUFFLE; + if (write_correct) { + cparams.filters[BLOSC2_MAX_FILTERS - 1] = 250; + } else { + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_FILTER_BYTEDELTA; + } + cparams.filters_meta[BLOSC2_MAX_FILTERS - 1] = 0; // 0 means typesize when using schunks + cparams.clevel = 9; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + if (cctx == NULL) { + printf("Cannot create compression context!\n"); return -1; } - /* Compress with clevel=5 and shuffle active */ - csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); - if (csize == 0) { - printf("Buffer is incompressible. Giving up.\n"); + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + if (cctx == NULL) { + printf("Cannot create decompression context!\n"); return -1; - } else if (csize < 0) { - printf("Compression error. Error code: %" PRId64 "\n", csize); - return (int) csize; } - csize_f += csize; - /* Decompress */ - dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); - if (dsize <= 0) { - printf("Decompression error. Error code: %" PRId64 "\n", dsize); - return (int) dsize; - } + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } - for (int i = 0; i < chunksize; i++) { - if (data_in[i] != data_dest[i]) { - printf("i: %d, data %u, dest %u", i, data_in[i], data_dest[i]); - printf("\n Decompressed data differs from original!\n"); + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, + chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is incompressible. Giving up.\n"); return -1; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + // Force usage of alternative decoder + if (read_correct) { + data_out[BLOSC2_CHUNK_FILTER_CODES + BLOSC2_MAX_FILTERS - 1] = 250; + } else { + data_out[BLOSC2_CHUNK_FILTER_CODES + BLOSC2_MAX_FILTERS - 1] = BLOSC_FILTER_BYTEDELTA; + } + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, + data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + + for (int i = 0; i < chunksize; i++) { + if (data_in[i] != data_dest[i]) { + printf("i: %d, data %u, dest %u", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } } } - } - csize_f = csize_f / nchunks; + csize_f = csize_f / nchunks; + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + } free(data_in); free(data_out); free(data_dest); - blosc2_free_ctx(cctx); - blosc2_free_ctx(dctx); printf("Successful roundtrip!\n"); printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double) csize_f); @@ -143,8 +247,8 @@ int rand_() { blosc2_storage b2_storage = {.cparams=&cparams}; b2_storage.contiguous = true; - b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape, chunkshape, blockshape, NULL, 0, - NULL, 0); + b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape, chunkshape, blockshape, NULL, + 0,NULL, 0); b2nd_array_t *arr; BLOSC_ERROR(b2nd_from_cbuffer(ctx, &arr, data, size)); @@ -180,8 +284,8 @@ int mixed_values() { blosc2_storage b2_storage = {.cparams=&cparams}; b2_storage.contiguous = true; - b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape, chunkshape, blockshape, NULL, 0, - NULL, 0); + b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape, chunkshape, blockshape, + NULL, 0, NULL, 0); b2nd_array_t *arr; BLOSC_ERROR(b2nd_from_cbuffer(ctx, &arr, data, size)); @@ -217,8 +321,8 @@ int arange_like() { blosc2_storage b2_storage = {.cparams=&cparams}; b2_storage.contiguous = true; - b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape, chunkshape, blockshape, NULL, 0, - NULL, 0); + b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape, chunkshape, blockshape, + NULL, 0,NULL, 0); b2nd_array_t *arr; BLOSC_ERROR(b2nd_from_cbuffer(ctx, &arr, data, size)); diff --git a/src/c-blosc2/plugins/filters/filters-registry.c b/src/c-blosc2/plugins/filters/filters-registry.c index 6a7ee5c4..04b69ac9 100644 --- a/src/c-blosc2/plugins/filters/filters-registry.c +++ b/src/c-blosc2/plugins/filters/filters-registry.c @@ -1,14 +1,15 @@ /* - Copyright (c) 2021 The Blosc Development Team + Copyright (c) 2021 The Blosc Development Team https://blosc.org License: BSD 3-Clause (see LICENSE.txt) */ -#include "blosc-private.h" #include "blosc2/filters-registry.h" #include "ndmean/ndmean.h" #include "ndcell/ndcell.h" #include "bytedelta/bytedelta.h" +#include "blosc-private.h" +#include "blosc2.h" void register_filters(void) { @@ -16,23 +17,33 @@ void register_filters(void) { ndcell.id = BLOSC_FILTER_NDCELL; ndcell.name = "ndcell"; ndcell.version = 1; - ndcell.forward = (blosc2_filter_forward_cb) ndcell_forward; - ndcell.backward = (blosc2_filter_backward_cb) ndcell_backward; + ndcell.forward = &ndcell_forward; + ndcell.backward = &ndcell_backward; register_filter_private(&ndcell); blosc2_filter ndmean; ndmean.id = BLOSC_FILTER_NDMEAN; ndmean.name = "ndmean"; ndmean.version = 1; - ndmean.forward = (blosc2_filter_forward_cb) ndmean_forward; - ndmean.backward = (blosc2_filter_backward_cb) ndmean_backward; + ndmean.forward = &ndmean_forward; + ndmean.backward = &ndmean_backward; register_filter_private(&ndmean); + // Buggy version. See #524 + blosc2_filter bytedelta_buggy; + bytedelta_buggy.id = BLOSC_FILTER_BYTEDELTA_BUGGY; + bytedelta_buggy.name = "bytedelta_buggy"; + bytedelta_buggy.version = 1; + bytedelta_buggy.forward = &bytedelta_forward_buggy; + bytedelta_buggy.backward = &bytedelta_backward_buggy; + register_filter_private(&bytedelta_buggy); + + // Fixed version. See #524 blosc2_filter bytedelta; bytedelta.id = BLOSC_FILTER_BYTEDELTA; bytedelta.name = "bytedelta"; - bytedelta.forward = (blosc2_filter_forward_cb) bytedelta_forward; - bytedelta.backward = (blosc2_filter_backward_cb) bytedelta_backward; + bytedelta.version = 1; + bytedelta.forward = &bytedelta_forward; + bytedelta.backward = &bytedelta_backward; register_filter_private(&bytedelta); - } diff --git a/src/c-blosc2/plugins/filters/ndcell/ndcell.c b/src/c-blosc2/plugins/filters/ndcell/ndcell.c index 0f251556..78a3ca2a 100644 --- a/src/c-blosc2/plugins/filters/ndcell/ndcell.c +++ b/src/c-blosc2/plugins/filters/ndcell/ndcell.c @@ -8,12 +8,15 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include #include "ndcell.h" +#include "../plugins/plugin_utils.h" +#include "blosc2/filters-registry.h" +#include "blosc2.h" + #include +#include +#include #include -#include "../plugins/plugin_utils.h" -#include "../include/blosc2/filters-registry.h" int ndcell_forward(const uint8_t *input, uint8_t *output, int32_t length, uint8_t meta, blosc2_cparams *cparams, diff --git a/src/c-blosc2/plugins/filters/ndcell/ndcell.h b/src/c-blosc2/plugins/filters/ndcell/ndcell.h index 34d44e26..8ffc49cc 100644 --- a/src/c-blosc2/plugins/filters/ndcell/ndcell.h +++ b/src/c-blosc2/plugins/filters/ndcell/ndcell.h @@ -8,10 +8,10 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef B2ND_NDCELL_H -#define B2ND_NDCELL_H +#ifndef BLOSC_PLUGINS_FILTERS_NDCELL_NDCELL_H +#define BLOSC_PLUGINS_FILTERS_NDCELL_NDCELL_H -#include +#include "blosc2.h" #define NDCELL_MAX_DIM 8 @@ -20,6 +20,4 @@ int ndcell_forward(const uint8_t* input, uint8_t* output, int32_t length, uint8_ int ndcell_backward(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_dparams* dparams, uint8_t id); -#endif //B2ND_NDCELL_H - - +#endif /* BLOSC_PLUGINS_FILTERS_NDCELL_NDCELL_H */ diff --git a/src/c-blosc2/plugins/filters/ndcell/test_ndcell.c b/src/c-blosc2/plugins/filters/ndcell/test_ndcell.c index e247a47c..25e91d39 100644 --- a/src/c-blosc2/plugins/filters/ndcell/test_ndcell.c +++ b/src/c-blosc2/plugins/filters/ndcell/test_ndcell.c @@ -29,12 +29,15 @@ **********************************************************************/ -#include #include "ndcell.h" -#include #include "blosc2/filters-registry.h" #include "b2nd.h" +#include +#include +#include +#include + static int test_ndcell(blosc2_schunk *schunk) { int64_t nchunks = schunk->nchunks; diff --git a/src/c-blosc2/plugins/filters/ndmean/ndmean.c b/src/c-blosc2/plugins/filters/ndmean/ndmean.c index 79f7408c..39b50262 100644 --- a/src/c-blosc2/plugins/filters/ndmean/ndmean.c +++ b/src/c-blosc2/plugins/filters/ndmean/ndmean.c @@ -6,10 +6,13 @@ #include "ndmean.h" -#include -#include #include "../plugins/plugin_utils.h" #include "../include/blosc2/filters-registry.h" +#include "blosc2/blosc2-common.h" + +#include +#include +#include int ndmean_forward(const uint8_t *input, uint8_t *output, int32_t length, uint8_t meta, blosc2_cparams *cparams, uint8_t id) { diff --git a/src/c-blosc2/plugins/filters/ndmean/ndmean.h b/src/c-blosc2/plugins/filters/ndmean/ndmean.h index 936e4105..e156cb47 100644 --- a/src/c-blosc2/plugins/filters/ndmean/ndmean.h +++ b/src/c-blosc2/plugins/filters/ndmean/ndmean.h @@ -8,8 +8,8 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef B2ND_NDMEAN_H -#define B2ND_NDMEAN_H +#ifndef BLOSC_PLUGINS_FILTERS_NDMEAN_NDMEAN_H +#define BLOSC_PLUGINS_FILTERS_NDMEAN_NDMEAN_H #include "blosc2.h" @@ -20,6 +20,4 @@ int ndmean_forward(const uint8_t* input, uint8_t* output, int32_t length, uint8_ int ndmean_backward(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_dparams* dparams, uint8_t id); -#endif //B2ND_NDMEAN_H - - +#endif /* BLOSC_PLUGINS_FILTERS_NDMEAN_NDMEAN_H */ diff --git a/src/c-blosc2/plugins/filters/ndmean/test_ndmean_mean.c b/src/c-blosc2/plugins/filters/ndmean/test_ndmean_mean.c index 7f2b5053..c2a61555 100644 --- a/src/c-blosc2/plugins/filters/ndmean/test_ndmean_mean.c +++ b/src/c-blosc2/plugins/filters/ndmean/test_ndmean_mean.c @@ -33,14 +33,15 @@ **********************************************************************/ -#include #include "ndmean.h" -#include -#include #include "blosc2/filters-registry.h" #include "../plugins/plugin_utils.h" #include "b2nd.h" +#include +#include +#include + #define EPSILON (float) (1) static bool is_close(double d1, double d2) { diff --git a/src/c-blosc2/plugins/filters/ndmean/test_ndmean_repart.c b/src/c-blosc2/plugins/filters/ndmean/test_ndmean_repart.c index 3dc4e352..c913deb6 100644 --- a/src/c-blosc2/plugins/filters/ndmean/test_ndmean_repart.c +++ b/src/c-blosc2/plugins/filters/ndmean/test_ndmean_repart.c @@ -31,13 +31,14 @@ **********************************************************************/ -#include #include "ndmean.h" -#include -#include #include "blosc2/filters-registry.h" #include "b2nd.h" +#include +#include +#include + #define EPSILON (float) (1e-5) static bool is_close(double d1, double d2) { diff --git a/src/c-blosc2/plugins/plugin_utils.c b/src/c-blosc2/plugins/plugin_utils.c index 4862b173..54974cb3 100644 --- a/src/c-blosc2/plugins/plugin_utils.c +++ b/src/c-blosc2/plugins/plugin_utils.c @@ -7,8 +7,7 @@ #include "plugin_utils.h" #include "blosc-private.h" #include "blosc2.h" - -#include +#include #define BLOSC_PLUGINS_MAX_DIM 8 diff --git a/src/c-blosc2/plugins/plugin_utils.h b/src/c-blosc2/plugins/plugin_utils.h index 085a6945..86a0c9d1 100644 --- a/src/c-blosc2/plugins/plugin_utils.h +++ b/src/c-blosc2/plugins/plugin_utils.h @@ -4,7 +4,12 @@ License: BSD 3-Clause (see LICENSE.txt) */ +#ifndef BLOSC_PLUGINS_PLUGIN_UTILS_H +#define BLOSC_PLUGINS_PLUGIN_UTILS_H + #include int32_t deserialize_meta(uint8_t *smeta, int32_t smeta_len, int8_t *ndim, int64_t *shape, int32_t *chunkshape, int32_t *blockshape); + +#endif /* BLOSC_PLUGINS_PLUGIN_UTILS_H */ diff --git a/src/c-blosc2/plugins/tuners/CMakeLists.txt b/src/c-blosc2/plugins/tuners/CMakeLists.txt new file mode 100644 index 00000000..739ffa5e --- /dev/null +++ b/src/c-blosc2/plugins/tuners/CMakeLists.txt @@ -0,0 +1 @@ +set(SOURCES ${SOURCES} ${PROJECT_SOURCE_DIR}/plugins/tuners/tuners-registry.c PARENT_SCOPE) diff --git a/src/c-blosc2/plugins/tunes/tunes-registry.c b/src/c-blosc2/plugins/tuners/tuners-registry.c similarity index 91% rename from src/c-blosc2/plugins/tunes/tunes-registry.c rename to src/c-blosc2/plugins/tuners/tuners-registry.c index 88f6efd0..2d734fd8 100644 --- a/src/c-blosc2/plugins/tunes/tunes-registry.c +++ b/src/c-blosc2/plugins/tuners/tuners-registry.c @@ -6,6 +6,9 @@ #include "blosc2/tuners-registry.h" #include "blosc-private.h" +#include "blosc2.h" + +#include void register_tuners(void) { diff --git a/src/c-blosc2/plugins/tunes/CMakeLists.txt b/src/c-blosc2/plugins/tunes/CMakeLists.txt deleted file mode 100644 index 93a6f3a9..00000000 --- a/src/c-blosc2/plugins/tunes/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -set(SOURCES ${SOURCES} ${PROJECT_SOURCE_DIR}/plugins/tunes/tunes-registry.c PARENT_SCOPE) diff --git a/src/c-blosc2/tests/b2nd/cutest.h b/src/c-blosc2/tests/b2nd/cutest.h index 379dec64..cdc70744 100644 --- a/src/c-blosc2/tests/b2nd/cutest.h +++ b/src/c-blosc2/tests/b2nd/cutest.h @@ -27,18 +27,17 @@ #define CUTEST_DATA(...) __VA_ARGS__ - -#define CUTEST_PARAMETRIZE(name, type, ...) \ - do { \ - type cutest_##name[] = {__VA_ARGS__}; \ - _cutest_parametrize(#name, cutest_##name, \ - sizeof(cutest_##name) / sizeof(type), sizeof(type)); \ +#define CUTEST_PARAMETRIZE(name, type, ...) \ + do { \ + type cutest_##name[] = {__VA_ARGS__}; \ + _cutest_parametrize(#name, cutest_##name, \ + sizeof(cutest_##name) / sizeof(type), sizeof(type)); \ } while(0) -#define CUTEST_PARAMETRIZE2(name, type, params_len, params) \ - do { \ - (type) *cutest_##name = params; \ - _cutest_parametrize(#name, cutest_##name, params_len, sizeof(type)); \ +#define CUTEST_PARAMETRIZE2(name, type, params_len, params) \ + do { \ + (type) *cutest_##name = params; \ + _cutest_parametrize(#name, cutest_##name, params_len, sizeof(type)); \ } while(0) #define CUTEST_GET_PARAMETER(name, type) \ @@ -50,29 +49,31 @@ #define CUTEST_TEST_TEARDOWN(sname) \ void sname##_teardown() -#define CUTEST_TEST_TEST(sname) \ - CUTEST_TEST_SETUP(sname); \ - CUTEST_TEST_TEARDOWN(sname); \ - int sname##_test(); \ - int sname##_test() \ - - -#define CUTEST_TEST_RUN(sname) \ - _cutest_setup(); \ - sname##_setup(); \ - int rc = _cutest_run((int (*)(void)) sname##_test, \ - #sname); \ - sname##_teardown(); \ - _cutest_teardown(); \ - return rc; - - -#define CUTEST_ASSERT(msg, cond) \ - do { \ - if (!(cond)) { \ - sprintf(_cutest_error_msg, "Error: %s %s:%d", msg, __FILE__, __LINE__); \ - return CUNIT_FAIL; \ - } \ + +#define CUTEST_TEST_TEST(sname) \ + CUTEST_TEST_SETUP(sname); \ + CUTEST_TEST_TEARDOWN(sname); \ + int sname##_test(); \ + int sname##_test() + + +#define CUTEST_TEST_RUN(sname) \ + do { \ + _cutest_setup(); \ + sname##_setup(); \ + int rc = _cutest_run((int (*)(void)) sname##_test, \ + #sname); \ + sname##_teardown(); \ + _cutest_teardown(); \ + return rc; \ + } while(0) + +#define CUTEST_ASSERT(msg, cond) \ + do { \ + if (!(cond)) { \ + sprintf(_cutest_error_msg, "Error: %s %s:%d", msg, __FILE__, __LINE__); \ + return CUNIT_FAIL; \ + } \ } while(0) diff --git a/src/c-blosc2/tests/b2nd/test_b2nd_open_offset.c b/src/c-blosc2/tests/b2nd/test_b2nd_open_offset.c index d29c105a..2b25a35a 100644 --- a/src/c-blosc2/tests/b2nd/test_b2nd_open_offset.c +++ b/src/c-blosc2/tests/b2nd/test_b2nd_open_offset.c @@ -67,7 +67,10 @@ CUTEST_TEST_TEST(open_offset) { data2[i] = 2 * i * nchunk; } nchunks = blosc2_schunk_append_buffer(schunk_write_start, data1, isize); - assert(nchunks == nchunk + 1); + if (nchunks != nchunk + 1) { + printf("Unexpected nchunks: %ld, %d\n", (long)nchunks, nchunk + 1); + return -1; + } blosc2_schunk_append_buffer(schunk_write_append, data2, isize); } @@ -130,7 +133,7 @@ CUTEST_TEST_TEST(open_offset) { blosc_set_timestamp(¤t); ttotal = blosc_elapsed_secs(last, current); printf("Time for fileframe (%s) + offset %ld -> open_offset : %.3g s, %.1f GB/s\n", - arr_read_offset->sc->storage->urlpath, offset, ttotal, (double)arr_read_offset->sc->nbytes / (ttotal * GB)); + arr_read_offset->sc->storage->urlpath, (long)offset, ttotal, (double)arr_read_offset->sc->nbytes / (ttotal * GB)); uint8_t* cframe_read_start, *cframe_read_offset; bool cframe_needs_free2, cframe_needs_free3; @@ -191,5 +194,5 @@ CUTEST_TEST_TEARDOWN(open_offset) { int main() { - CUTEST_TEST_RUN(open_offset) + CUTEST_TEST_RUN(open_offset); } diff --git a/src/c-blosc2/tests/b2nd/test_common.h b/src/c-blosc2/tests/b2nd/test_common.h index 54a29961..e7fef9d2 100644 --- a/src/c-blosc2/tests/b2nd/test_common.h +++ b/src/c-blosc2/tests/b2nd/test_common.h @@ -8,8 +8,8 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef B2ND_TEST_COMMON_H -#define B2ND_TEST_COMMON_H +#ifndef BLOSC_TESTS_B2ND_TEST_COMMON_H +#define BLOSC_TESTS_B2ND_TEST_COMMON_H #include #include "context.h" @@ -96,4 +96,4 @@ void b2nd_default_parameters() { } -#endif //B2ND_TEST_COMMON_H +#endif /* BLOSC_TESTS_B2ND_TEST_COMMON_H */ diff --git a/src/c-blosc2/tests/cutest.h b/src/c-blosc2/tests/cutest.h index 1aaaff46..d8f9ec38 100644 --- a/src/c-blosc2/tests/cutest.h +++ b/src/c-blosc2/tests/cutest.h @@ -8,8 +8,8 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef CUTEST_CUTEST_H -#define CUTEST_CUTEST_H +#ifndef BLOSC_TESTS_CUTEST_H +#define BLOSC_TESTS_CUTEST_H #include #include @@ -28,18 +28,17 @@ #define CUTEST_DATA(...) __VA_ARGS__ - -#define CUTEST_PARAMETRIZE(name, type, ...) \ - do { \ - type cutest_##name[] = {__VA_ARGS__}; \ - _cutest_parametrize(#name, cutest_##name, \ - sizeof(cutest_##name) / sizeof(type), sizeof(type)); \ +#define CUTEST_PARAMETRIZE(name, type, ...) \ + do { \ + type cutest_##name[] = {__VA_ARGS__}; \ + _cutest_parametrize(#name, cutest_##name, \ + sizeof(cutest_##name) / sizeof(type), sizeof(type)); \ } while(0) -#define CUTEST_PARAMETRIZE2(name, type, params_len, params) \ - do { \ - (type) *cutest_##name = params; \ - _cutest_parametrize(#name, cutest_##name, params_len, sizeof(type)); \ +#define CUTEST_PARAMETRIZE2(name, type, params_len, params) \ + do { \ + (type) *cutest_##name = params; \ + _cutest_parametrize(#name, cutest_##name, params_len, sizeof(type)); \ } while(0) #define CUTEST_GET_PARAMETER(name, type) \ @@ -54,23 +53,25 @@ #define CUTEST_TEST_TEARDOWN(sname) \ void sname##_teardown(struct sname##_data *data) -#define CUTEST_TEST_TEST(sname) \ +#define CUTEST_TEST_TEST(sname) \ static struct sname##_data test_##sname##_data; \ CUTEST_TEST_SETUP(sname); \ CUTEST_TEST_TEARDOWN(sname); \ int sname##_test(struct sname##_data* data); \ - int sname##_test(struct sname##_data* data) \ + int sname##_test(struct sname##_data* data) #define CUTEST_TEST_RUN(sname) \ - _cutest_setup(); \ - sname##_setup(&test_##sname##_data); \ - int rc = _cutest_run((int (*)(void *)) sname##_test, \ - (void *) &test_##sname##_data, \ - #sname); \ - sname##_teardown(&test_##sname##_data); \ - _cutest_teardown(); \ - return rc; + do { \ + _cutest_setup(); \ + sname##_setup(&test_##sname##_data); \ + int rc = _cutest_run((int (*)(void *)) sname##_test, \ + (void *) &test_##sname##_data, \ + #sname); \ + sname##_teardown(&test_##sname##_data); \ + _cutest_teardown(); \ + return rc; \ + } while(0) #define CUTEST_ASSERT(msg, cond) \ @@ -202,5 +203,4 @@ int _cutest_run(int (*test)(void *), void *test_data, char *name) { return cutest_failed; } - -#endif //CUTEST_CUTEST_H +#endif /* BLOSC_TESTS_CUTEST_H */ diff --git a/src/c-blosc2/tests/fuzz/CMakeLists.txt b/src/c-blosc2/tests/fuzz/CMakeLists.txt index 96417dd7..d8a4aea0 100644 --- a/src/c-blosc2/tests/fuzz/CMakeLists.txt +++ b/src/c-blosc2/tests/fuzz/CMakeLists.txt @@ -1,6 +1,3 @@ -# flags -link_directories(${PROJECT_BINARY_DIR}/blosc) - # look for fuzzing lib and link with it if found if(CMAKE_C_COMPILER_ID STREQUAL "Clang") enable_language(CXX) @@ -64,11 +61,10 @@ foreach(source ${SOURCES}) if(FUZZING_ENGINE_FOUND) set_target_properties(${target} PROPERTIES LINKER_LANGUAGE CXX) - target_link_libraries(${target} ${FUZZING_ENGINE}) + target_link_libraries(${target} PUBLIC ${FUZZING_ENGINE}) endif() - target_link_libraries(${target} blosc2_static) - add_dependencies(${target} blosc2_static) + target_link_libraries(${target} PUBLIC blosc2_static) # run standalone fuzzer against each file file(GLOB COMPAT_FILES ${PROJECT_SOURCE_DIR}/compat/*.cdata) diff --git a/src/c-blosc2/tests/test_common.h b/src/c-blosc2/tests/test_common.h index 0b277025..3525b6bc 100644 --- a/src/c-blosc2/tests/test_common.h +++ b/src/c-blosc2/tests/test_common.h @@ -10,8 +10,8 @@ See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_TEST_COMMON_H -#define BLOSC_TEST_COMMON_H +#ifndef BLOSC_TESTS_TEST_COMMON_H +#define BLOSC_TESTS_TEST_COMMON_H #include "blosc2.h" @@ -32,7 +32,7 @@ #if defined(_WIN32) /* MSVC does not have setenv */ - #define setenv(name, value, overwrite) do {_putenv_s(name, value);} while(0) + #define setenv(name, value, overwrite) (_putenv_s(name, value)) #endif @@ -59,9 +59,9 @@ extern int tests_run; /** Allocates a block of memory with the specified size and alignment. The allocated memory is 'cleaned' before returning to avoid - accidental re-use of data within or between tests. + accidental reuse of data within or between tests. */ -inline static void* blosc_test_malloc(const size_t alignment, const size_t size) { +static inline void* blosc_test_malloc(const size_t alignment, const size_t size) { const int32_t clean_value = 0x99; void* block = NULL; int32_t res = 0; @@ -95,7 +95,7 @@ inline static void* blosc_test_malloc(const size_t alignment, const size_t size) } /** Frees memory allocated by blosc_test_malloc. */ -inline static void blosc_test_free(void* ptr) { +static inline void blosc_test_free(void* ptr) { #if defined(_WIN32) _aligned_free(ptr); #else @@ -104,7 +104,7 @@ inline static void blosc_test_free(void* ptr) { } /** Fills a buffer with contiguous values. */ -inline static void blosc_test_fill_seq(void* const ptr, const size_t size) { +static inline void blosc_test_fill_seq(void* const ptr, const size_t size) { size_t k; uint8_t* const byte_ptr = (uint8_t*)ptr; for (k = 0; k < size; k++) { @@ -113,7 +113,7 @@ inline static void blosc_test_fill_seq(void* const ptr, const size_t size) { } /** Fills a buffer with random values. */ -inline static void blosc_test_fill_random(void* const ptr, const size_t size) { +static inline void blosc_test_fill_random(void* const ptr, const size_t size) { size_t k; uint8_t* const byte_ptr = (uint8_t*)ptr; for (k = 0; k < size; k++) { @@ -126,7 +126,7 @@ inline static void blosc_test_fill_random(void* const ptr, const size_t size) { */ /** Parse a `int32_t` value from a string, checking for overflow. */ -inline static bool blosc_test_parse_uint32_t(const char* const str, uint32_t* value) { +static inline bool blosc_test_parse_uint32_t(const char* const str, uint32_t* value) { char* str_end; long signed_value = strtol(str, &str_end, 10); if (signed_value < 0 || *str_end) { @@ -144,7 +144,7 @@ inline static bool blosc_test_parse_uint32_t(const char* const str, uint32_t* va /** Print an error message when a test program has been invoked with an invalid number of arguments. */ -inline static void blosc_test_print_bad_argcount_msg( +static inline void blosc_test_print_bad_argcount_msg( const int32_t num_expected_args, const int32_t num_actual_args) { fprintf(stderr, "Invalid number of arguments specified.\nExpected %d arguments but was given %d.", num_expected_args, num_actual_args); @@ -152,7 +152,7 @@ inline static void blosc_test_print_bad_argcount_msg( /** Print an error message when a test program has been invoked with an invalid argument value. */ -inline static void blosc_test_print_bad_arg_msg(const int32_t arg_index) { +static inline void blosc_test_print_bad_arg_msg(const int32_t arg_index) { fprintf(stderr, "Invalid value specified for argument at index %d.\n", arg_index); } @@ -167,7 +167,7 @@ static void dummy_threads_callback(void *callback_data, void (*dojob)(void *), i } /* install the callback if environment variable BLOSC_TEST_CALLBACK="yes" */ -inline static void install_blosc_callback_test(void) +static inline void install_blosc_callback_test(void) { char *callback_env; callback_env = getenv("BLOSC_TEST_CALLBACK"); @@ -175,4 +175,4 @@ inline static void install_blosc_callback_test(void) blosc2_set_threads_callback(dummy_threads_callback, NULL); } -#endif /* !defined(BLOSC_TEST_COMMON_H) */ +#endif /* BLOSC_TESTS_TEST_COMMON_H */ diff --git a/src/c-blosc2/tests/test_copy.c b/src/c-blosc2/tests/test_copy.c index 1aeef31c..9d6953a2 100644 --- a/src/c-blosc2/tests/test_copy.c +++ b/src/c-blosc2/tests/test_copy.c @@ -193,5 +193,5 @@ CUTEST_TEST_TEARDOWN(copy) { int main() { - CUTEST_TEST_RUN(copy) + CUTEST_TEST_RUN(copy); } diff --git a/src/c-blosc2/tests/test_fill_special.c b/src/c-blosc2/tests/test_fill_special.c index a8bd8a15..c87deeb5 100644 --- a/src/c-blosc2/tests/test_fill_special.c +++ b/src/c-blosc2/tests/test_fill_special.c @@ -173,5 +173,5 @@ CUTEST_TEST_TEARDOWN(fill_special) { int main() { - CUTEST_TEST_RUN(fill_special) + CUTEST_TEST_RUN(fill_special); } diff --git a/src/c-blosc2/tests/test_filters.c b/src/c-blosc2/tests/test_filters.c new file mode 100644 index 00000000..c65681bc --- /dev/null +++ b/src/c-blosc2/tests/test_filters.c @@ -0,0 +1,50 @@ +#include +#include +#include + +#define NCHUNKS 1 +#define TYPESIZE 2 +#define LEN 39 +#define CHUNKSIZE (TYPESIZE * LEN) + +int main(void) { + blosc2_init(); + srand(0); + if (blosc2_compname_to_compcode("zstd") < 0) { + // We need ZSTD for the test here... + return 0; + } + + uint16_t *ref_data = (uint16_t *)malloc(CHUNKSIZE); + uint16_t *data_dest = (uint16_t *)malloc(CHUNKSIZE); + for (int i = 0; i < LEN; i++) { + ref_data[i] = rand() % 118; + } + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.compcode = BLOSC_ZSTD; + cparams.filters[BLOSC2_MAX_FILTERS - 2] = BLOSC_BITSHUFFLE; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + + cparams.typesize = TYPESIZE; + blosc2_storage storage = {.contiguous=false, .urlpath=NULL, .cparams=&cparams, .dparams=&dparams}; + + blosc2_schunk* schunk = blosc2_schunk_new(&storage); + blosc2_schunk_append_buffer(schunk, ref_data, CHUNKSIZE); + + blosc2_schunk_decompress_chunk(schunk, 0, data_dest, CHUNKSIZE); + for (int i = 0; i < LEN; i++) { + if (data_dest[i] != ref_data[i]) { + printf("Decompressed data differs from original %d, %d, %d!\n", + i, ref_data[i], data_dest[i]); + return -1; + } + } + + printf("Successful roundtrip data <-> schunk !\n"); + + blosc2_schunk_free(schunk); + blosc2_destroy(); + return 0; +} diff --git a/src/c-blosc2/tests/test_frame_get_offsets.c b/src/c-blosc2/tests/test_frame_get_offsets.c index 8f1b76b6..fa582ba1 100644 --- a/src/c-blosc2/tests/test_frame_get_offsets.c +++ b/src/c-blosc2/tests/test_frame_get_offsets.c @@ -102,7 +102,13 @@ CUTEST_TEST_TEST(fill_special) { free(chunk); } for (int i = 1; i < schunk->nchunks; ++i) { - CUTEST_ASSERT("Error getting the offsets", (offsets[i] - offsets[i - 1]) == chunk_size); + dsize = blosc2_schunk_get_chunk(schunk, i-1, &chunk, &needs_free); + CUTEST_ASSERT("ERROR: chunk cannot be retrieved correctly.", dsize >= 0); + blosc2_cbuffer_sizes(chunk, &nbytes_, &cbytes_, &blocksize); + if (needs_free) { + free(chunk); + } + CUTEST_ASSERT("Error getting the offsets", (offsets[i] - offsets[i - 1]) == cbytes_); } } @@ -121,5 +127,5 @@ CUTEST_TEST_TEARDOWN(fill_special) { int main() { - CUTEST_TEST_RUN(fill_special) + CUTEST_TEST_RUN(fill_special); } diff --git a/src/c-blosc2/tests/test_frame_offset.c b/src/c-blosc2/tests/test_frame_offset.c index a6ae394d..338d52db 100644 --- a/src/c-blosc2/tests/test_frame_offset.c +++ b/src/c-blosc2/tests/test_frame_offset.c @@ -185,5 +185,5 @@ CUTEST_TEST_TEARDOWN(fill_special) { int main() { - CUTEST_TEST_RUN(fill_special) + CUTEST_TEST_RUN(fill_special); } diff --git a/src/c-blosc2/tests/test_noinit.c b/src/c-blosc2/tests/test_noinit.c index ff6d2833..3cae0d38 100644 --- a/src/c-blosc2/tests/test_noinit.c +++ b/src/c-blosc2/tests/test_noinit.c @@ -71,7 +71,10 @@ int main(void) { /* Launch several subprocesses */ for (i = 1; i <= nchildren; i++) { pid = fork(); - assert(pid >= 0); + if (pid < 0) { + printf("Error at fork()!"); + exit(1); + } } blosc2_set_nthreads(4); diff --git a/src/c-blosc2/tests/test_small_chunks.c b/src/c-blosc2/tests/test_small_chunks.c index 9e0a361d..172e647d 100644 --- a/src/c-blosc2/tests/test_small_chunks.c +++ b/src/c-blosc2/tests/test_small_chunks.c @@ -76,5 +76,5 @@ CUTEST_TEST_TEARDOWN(small_chunks) { int main() { - CUTEST_TEST_RUN(small_chunks) + CUTEST_TEST_RUN(small_chunks); } diff --git a/src/c-blosc2/tests/test_udio.c b/src/c-blosc2/tests/test_udio.c index c74bacbf..c6f9a4da 100644 --- a/src/c-blosc2/tests/test_udio.c +++ b/src/c-blosc2/tests/test_udio.c @@ -176,5 +176,5 @@ CUTEST_TEST_TEARDOWN(udio) { int main() { - CUTEST_TEST_RUN(udio) + CUTEST_TEST_RUN(udio); } diff --git a/src/c-blosc2/tests/test_urcodecs.c b/src/c-blosc2/tests/test_urcodecs.c index aad2a620..dc2cf109 100644 --- a/src/c-blosc2/tests/test_urcodecs.c +++ b/src/c-blosc2/tests/test_urcodecs.c @@ -263,5 +263,5 @@ CUTEST_TEST_TEARDOWN(urcodecs) { int main() { - CUTEST_TEST_RUN(urcodecs) + CUTEST_TEST_RUN(urcodecs); } diff --git a/src/c-blosc2/tests/test_urfilters.c b/src/c-blosc2/tests/test_urfilters.c index 01bad57a..3cf23c11 100644 --- a/src/c-blosc2/tests/test_urfilters.c +++ b/src/c-blosc2/tests/test_urfilters.c @@ -265,5 +265,5 @@ CUTEST_TEST_TEARDOWN(urfilters) { int main() { - CUTEST_TEST_RUN(urfilters) + CUTEST_TEST_RUN(urfilters); } diff --git a/src/c-blosc2/tests/test_zero_runlen.c b/src/c-blosc2/tests/test_zero_runlen.c index 511dcfaf..a17c37ee 100644 --- a/src/c-blosc2/tests/test_zero_runlen.c +++ b/src/c-blosc2/tests/test_zero_runlen.c @@ -217,5 +217,5 @@ CUTEST_TEST_TEARDOWN(zero_runlen) { int main() { - CUTEST_TEST_RUN(zero_runlen) + CUTEST_TEST_RUN(zero_runlen); }