From df6f2fc565f3709a07bc7f6031c469cc8ae35480 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:04:04 +0200 Subject: [PATCH 1/9] Update LZ4 filter to latest commit available on github --- doc/information.rst | 2 +- src/LZ4/COPYING | 3 ++- src/LZ4/lz4_h5filter.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/information.rst b/doc/information.rst index 046b0b9a..acf2b561 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -68,7 +68,7 @@ HDF5 filters and compression libraries HDF5 compression filters and compression libraries sources were obtained from: * `LZ4 plugin `_ - (commit `d48f960 `_) + (commit `49e3b65 `_) using LZ4. * `bitshuffle plugin `_ (v0.5.1) using LZ4 and ZStd. * bzip2 plugin (from `PyTables `_ v3.9.2) diff --git a/src/LZ4/COPYING b/src/LZ4/COPYING index 41dffc9c..4c6315cd 100644 --- a/src/LZ4/COPYING +++ b/src/LZ4/COPYING @@ -1,3 +1,4 @@ + Copyright Notice and License Terms for HDF5 LZ4 compression filter plugin ----------------------------------------------------------------------------- @@ -56,4 +57,4 @@ Source code of the HDF5 LZ4 filter is based on h5py implementation found in Additional_Legal/h5py_Copyrights_and_Licenses.txt Original source: http://docs.h5py.org/en/latest/licenses.html -License type: BSD-style license + License type: BSD-style license diff --git a/src/LZ4/lz4_h5filter.h b/src/LZ4/lz4_h5filter.h index a0995a9b..c8d808e7 100644 --- a/src/LZ4/lz4_h5filter.h +++ b/src/LZ4/lz4_h5filter.h @@ -17,7 +17,7 @@ #ifndef LZ4_H5FILTER_H -#define LZ5_H5FILTER_H +#define LZ4_H5FILTER_H #define H5Z_class_t_vers 2 #include "hdf5.h" From 70a7c2bf008b3ab915374aa614d90014f210f592 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:06:59 +0200 Subject: [PATCH 2/9] Squashed 'src/hdf5-blosc/' changes from bd8ee59..949947a 949947a Update release notes for 1.0.1 e2f7493 Merge pull request #17 from zayfod/master aaae50a Merge pull request #33 from lgarrison/fix-cmake-libdir ba20858 cmake: fix build on systems where the libdir is lib64 9683f7d Merge pull request #24 from bhawkins/fix-link c0d81ef Merge pull request #32 from matchy233/fix/warnings cd04690 Merge pull request #31 from mkitti/master 526e3f4 Fix warnings related to const for blosc_filter.c b80cacd Fix branch name of c-blosc repository in CMakeLists ff32d8a Allow for HDF5_ROOT and set blosc tag to main ec0d1ab Fix link to HDF5 docs. 6d60f33 Added freeing of memory, allocated by register_blosc(). git-subtree-dir: src/hdf5-blosc git-subtree-split: 949947a5516bb987570c900c51866a7a50fb2dd7 --- CMakeLists.txt | 7 +++++-- README.rst | 2 +- RELEASE_NOTES.txt | 9 +++++++++ src/blosc_filter.c | 4 ++-- src/example.c | 2 ++ 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f3de7671..9ec0f23e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 2.8.10) +cmake_policy(SET CMP0074 NEW) project(blosc_hdf5) include(ExternalProject) +include(GNUInstallDirs) # options option(BUILD_TESTS @@ -9,7 +11,7 @@ option(BUILD_TESTS option(BUILD_PLUGIN "Build dynamically loadable plugin for HDF5 version > 1.8.11" ON) if(BUILD_PLUGIN) - set(PLUGIN_INSTALL_PATH "/usr/local/hdf5/lib/plugin" CACHE PATH + set(PLUGIN_INSTALL_PATH "/usr/local/hdf5/lib/plugin" CACHE PATH "Where to install the dynamic HDF5-plugin") endif(BUILD_PLUGIN) @@ -25,6 +27,7 @@ message("GIT_EXECUTABLE='${GIT_EXECUTABLE}'") ExternalProject_Add(project_blosc PREFIX ${BLOSC_PREFIX} GIT_REPOSITORY https://github.com/Blosc/c-blosc.git + GIT_TAG main INSTALL_DIR ${BLOSC_INSTALL_DIR} CMAKE_ARGS ${BLOSC_CMAKE_ARGS} ) @@ -52,7 +55,7 @@ include_directories(${HDF5_INCLUDE_DIRS}) # add blosc libraries add_library(blosc_shared SHARED IMPORTED) -set_property(TARGET blosc_shared PROPERTY IMPORTED_LOCATION ${BLOSC_INSTALL_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}blosc${CMAKE_SHARED_LIBRARY_SUFFIX}) +set_property(TARGET blosc_shared PROPERTY IMPORTED_LOCATION ${BLOSC_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}blosc${CMAKE_SHARED_LIBRARY_SUFFIX}) add_dependencies(blosc_shared project_blosc) include_directories(${BLOSC_INSTALL_DIR}/include) diff --git a/README.rst b/README.rst index 671c1e1f..7ca47384 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ Instead of just linking this Blosc filter into your HDF5 application, it is poss it as a system-wide HDF5 plugin (with HDF5 1.8.11 or later). This is useful because it allows *every* HDF5-using program on your system to transparently read Blosc-compressed HDF5 files. -As described in the `HDF5 plugin documentation `_, you just need to compile the Blosc plugin into a shared library and +As described in the `HDF5 plugin documentation `_, you just need to compile the Blosc plugin into a shared library and copy it to the plugin directory (which defaults to ``/usr/local/hdf5/lib/plugin`` on non-Windows systems). Following the ``cmake`` instructions below produces a ``libH5Zblosc.so`` shared library diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index d260d37f..fcbab152 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -6,6 +6,15 @@ :Contact: francesc@blosc.org :URL: http://www.blosc.org +Changes from 1.0.0 to 1.0.1 +=========================== + +- Fix warnings related to const for blosc_filter.c. Thanks to + @matchy233. Closes #34. + +- Fix build on systems where the libdir is lib64 instead of lib. Thanks to + Lehman Garrison. + Changes from 0.0.1 to 1.0.0 =========================== diff --git a/src/blosc_filter.c b/src/blosc_filter.c index bfd8c3eb..a099e2d3 100644 --- a/src/blosc_filter.c +++ b/src/blosc_filter.c @@ -155,8 +155,8 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts, int doshuffle = 1; /* Shuffle default */ int compcode; /* Blosc compressor */ int code; - char* compname = "blosclz"; /* The compressor by default */ - char* complist; + const char* compname = "blosclz"; /* The compressor by default */ + const char* complist; char errmsg[256]; /* Filter params that are always set */ diff --git a/src/example.c b/src/example.c index f06d4119..0e0f184a 100644 --- a/src/example.c +++ b/src/example.c @@ -56,6 +56,8 @@ int main(){ /* Register the filter with the library */ r = register_blosc(&version, &date); printf("Blosc version info: %s (%s)\n", version, date); + free(version); + free(date); if(r<0) goto failed; From e475b44796b59e8adf234e1a3c622b37c2659f05 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:08:06 +0200 Subject: [PATCH 3/9] Update hdf5-blosc version in documentation --- doc/information.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/information.rst b/doc/information.rst index acf2b561..d67fcb00 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -73,7 +73,7 @@ HDF5 compression filters and compression libraries sources were obtained from: * `bitshuffle plugin `_ (v0.5.1) using LZ4 and ZStd. * bzip2 plugin (from `PyTables `_ v3.9.2) using `BZip2 `_ (v1.0.8). -* `hdf5-blosc plugin `_ (v1.0.0) +* `hdf5-blosc plugin `_ (v1.0.1) using `c-blosc `_ (v1.21.5), LZ4, Snappy, ZLib and ZStd. * hdf5-blosc2 plugin (from `PyTables `_ v3.9.2) using `c-blosc2 `_ (v2.13.2), LZ4, ZLib and ZStd. From b86055454903bcac6e9f0958075c4d8197e23d61 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:30:37 +0200 Subject: [PATCH 4/9] Squashed 'src/SZ/' changes from c25805c..f466775 f466775 Merge pull request #116 from ayzk/master1 bde9259 fix a bug when dimension is corrected in compression but not in decompression a92658e Merge pull request #109 from vasole/master a5d6039 Restrict the scope only to windows without MINGW as already done. d0c7f6c Force static build of external libraries with MSVC 73065ab Remove gcc specific options git-subtree-dir: src/SZ git-subtree-split: f4667759ead6a902110e80ff838ccdfddbc8dcd7 --- sz/CMakeLists.txt | 20 ++++++++++++++++---- sz/src/sz.c | 11 ++++++++++- zlib/CMakeLists.txt | 4 ++++ zstd/CMakeLists.txt | 4 ++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/sz/CMakeLists.txt b/sz/CMakeLists.txt index 7fb9bfab..86eae0fc 100644 --- a/sz/CMakeLists.txt +++ b/sz/CMakeLists.txt @@ -92,11 +92,23 @@ set_target_properties (SZ PROPERTIES OUTPUT_NAME_RELWITHDEBINFO ${LIB_RELEASE_NAME} ) -target_link_libraries (SZ PUBLIC ${ZLIB_dep} ${ZSTD_dep} m) +if (WIN32 AND NOT MINGW) + target_link_libraries (SZ PUBLIC ${ZLIB_dep} ${ZSTD_dep}) +else () + target_link_libraries (SZ PUBLIC ${ZLIB_dep} ${ZSTD_dep} m) +endif () -target_compile_options(SZ - PRIVATE $<$:-Wall -Wextra -Wpedantic -Wno-unused-parameter> - ) +if (WIN32 AND NOT MINGW) + message(STATUS "Native windows compiler detected, adding compile flags") + target_compile_options(SZ + PRIVATE $<$:-Wall> + ) +else () + message(STATUS "GCC detected, adding compile flags") + target_compile_options(SZ + PRIVATE $<$:-Wall -Wextra -Wpedantic -Wno-unused-parameter> + ) +endif() if(BUILD_OPENMP) target_link_libraries(SZ PRIVATE OpenMP::OpenMP_C) diff --git a/sz/src/sz.c b/sz/src/sz.c index 51e98b8a..6bb789bb 100644 --- a/sz/src/sz.c +++ b/sz/src/sz.c @@ -485,7 +485,16 @@ unsigned char *SZ_compress_rev(int dataType, void *data, void *reservedValue, si void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - if(confparams_dec==NULL) + //correct dimension if needed + size_t _r[5]; + filterDimension(r5, r4, r3, r2, r1, _r); + r5 = _r[4]; + r4 = _r[3]; + r3 = _r[2]; + r2 = _r[1]; + r1 = _r[0]; + + if(confparams_dec==NULL) confparams_dec = (sz_params*)malloc(sizeof(sz_params)); memset(confparams_dec, 0, sizeof(sz_params)); if(exe_params==NULL) diff --git a/zlib/CMakeLists.txt b/zlib/CMakeLists.txt index 305a1e2e..09eb7178 100644 --- a/zlib/CMakeLists.txt +++ b/zlib/CMakeLists.txt @@ -4,6 +4,10 @@ else () set (BUILD_EXT_LIBS_TYPE "STATIC") endif () +if (WIN32 AND NOT MINGW) + set (BUILD_EXT_LIBS_TYPE "STATIC") +endif () + add_library(ZLIB ${BUILD_EXT_LIBS_TYPE} ./gzclose.c ./uncompr.c diff --git a/zstd/CMakeLists.txt b/zstd/CMakeLists.txt index 9a6bc395..ea13dba6 100644 --- a/zstd/CMakeLists.txt +++ b/zstd/CMakeLists.txt @@ -4,6 +4,10 @@ else () set (BUILD_EXT_LIBS_TYPE "STATIC") endif () +if (WIN32 AND NOT MINGW) + set (BUILD_EXT_LIBS_TYPE "STATIC") +endif () + add_library(zstd ${BUILD_EXT_LIBS_TYPE} ./common/entropy_common.c ./common/pool.c From a7e9f79b6053aeaa660954b0c43b949224531aec Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:32:44 +0200 Subject: [PATCH 5/9] Update SZ commit in documentation --- doc/information.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/information.rst b/doc/information.rst index d67fcb00..42eeeff9 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -81,7 +81,7 @@ HDF5 compression filters and compression libraries sources were obtained from: using `CharLS `_ (1.x branch, commit `25160a4 `_). * `SZ plugin `_ - (commit `c25805c12b3 `_) + (commit `f466775 `_) * `H5Z-SPERR plugin `_ (v0.1.3) using `SPERR `_ (v0.8.1). using `SZ `_, ZLib and ZStd. * `SZ3 plugin `_ From c7d77b43f5f9555f6eeeb067f165f0596256cf0f Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:33:25 +0200 Subject: [PATCH 6/9] Change link of LZ4 commit --- doc/information.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/information.rst b/doc/information.rst index 42eeeff9..a7fe9cf3 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -68,7 +68,7 @@ HDF5 filters and compression libraries HDF5 compression filters and compression libraries sources were obtained from: * `LZ4 plugin `_ - (commit `49e3b65 `_) + (commit `49e3b65 `_) using LZ4. * `bitshuffle plugin `_ (v0.5.1) using LZ4 and ZStd. * bzip2 plugin (from `PyTables `_ v3.9.2) From d1b6c02a01483cf83e0bb3564f97c46b1d8401d1 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:38:42 +0200 Subject: [PATCH 7/9] fix documentation of SZ ans SPERR filters dependencies --- doc/information.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/information.rst b/doc/information.rst index a7fe9cf3..4f8c470a 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -82,8 +82,8 @@ HDF5 compression filters and compression libraries sources were obtained from: (1.x branch, commit `25160a4 `_). * `SZ plugin `_ (commit `f466775 `_) -* `H5Z-SPERR plugin `_ (v0.1.3) using `SPERR `_ (v0.8.1). using `SZ `_, ZLib and ZStd. +* `H5Z-SPERR plugin `_ (v0.1.3) using `SPERR `_ (v0.8.1). * `SZ3 plugin `_ (commit `4bbe9df7e4bcb `_) using `SZ3 `_ and ZStd. From 3afef9b961ab887a7b2b3d2f3507a230150c08d7 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:46:55 +0200 Subject: [PATCH 8/9] Squashed 'src/snappy/' changes from dc05e026..2c94e111 2c94e111 Release version 1.2.1 465b5b60 Restore old compression functions to preserve ABI 52820ea9 Merge pull request #178 from jjerphan/build/update-version-to-1.2.0 ac6b63f0 Update version number to 1.2.0 23b32868 Merge pull request #175 from Quuxplusone/suggest-override 6b2eb702 Fix all compilation errors to be C++11 compliant ab38064a Fix compilation in the benchmark 4e693db1 Use C++11 style instead of C++20 a60fd602 Fix sync 766d24c9 Zippy level 2 for denser compression and faster decompression 4f5cf9a8 Internal changes 8bf26408 Internal changes f0b0c9b8 Internal changes 54d07d53 Restructure compression sampling for comparative analysis 41a3ade2 Silence -Wdeprecated warning on clang 8774875e Fix -Wsuggest-override warnings from Clang 27f34a58 Fix -Wsign-compare warning c9f9edf6 Fixes for Windows bazel build. 66a30b80 Add initial bazel build support for snappy. f725f676 Upgrade googletest to v1.13.0 release. 83253929 Disable Wimplicit-int-float-conversion warning in googletest 108139d2 Upgrade benchmark library to v1.7.1 release. 00aa9ac6 Disable -Wsign-compare warning. cfc573e0 Define missing SNAPPY_PREFETCH macros. 92f18e66 Add prefetch to zippy compress f603a020 Explicitly #include in snappy-internal.h 9c42b71b Optimize check for uncommon decompression for ARM, saving two instructions and three cycles. git-subtree-dir: src/snappy git-subtree-split: 2c94e11145f0b7b184b831577c93e5a41c4c0346 --- .bazelrc | 2 + .gitignore | 1 + BUILD.bazel | 210 ++++++++++++++++++++++++++++ CMakeLists.txt | 20 ++- WORKSPACE | 27 ++++ cmake/config.h.in | 3 + snappy-internal.h | 29 ++++ snappy-stubs-internal.h | 6 + snappy.cc | 284 +++++++++++++++++++++++++++++++++----- snappy.h | 47 ++++++- snappy_benchmark.cc | 56 +++++--- snappy_compress_fuzzer.cc | 32 +++-- third_party/benchmark | 2 +- third_party/googletest | 2 +- 14 files changed, 647 insertions(+), 74 deletions(-) create mode 100644 .bazelrc create mode 100644 BUILD.bazel create mode 100644 WORKSPACE diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 00000000..3843446b --- /dev/null +++ b/.bazelrc @@ -0,0 +1,2 @@ +# googletest requires C++14 or above +build --cxxopt='-std=c++17' diff --git a/.gitignore b/.gitignore index c4b24253..826af251 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ # Build directory. build/ +/bazel-* out/ diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 00000000..97c9f3aa --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,210 @@ +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +SNAPPY_VERSION = (1, 1, 10) + +config_setting( + name = "windows", + constraint_values = ["@platforms//os:windows"], +) + +cc_library( + name = "config", + hdrs = ["config.h"], + defines = ["HAVE_CONFIG_H"] +) + +cc_library( + name = "snappy-stubs-public", + hdrs = [":snappy-stubs-public.h"], +) + +cc_library( + name = "snappy-stubs-internal", + srcs = ["snappy-stubs-internal.cc"], + hdrs = ["snappy-stubs-internal.h"], + deps = [ + ":config", + ":snappy-stubs-public", + ], +) + +cc_library( + name = "snappy", + srcs = [ + "snappy.cc", + "snappy-internal.h", + "snappy-sinksource.cc", + ], + hdrs = [ + "snappy.h", + "snappy-sinksource.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": [ + "-Wno-sign-compare", + ]}), + deps = [ + ":config", + ":snappy-stubs-internal", + ":snappy-stubs-public", + ], +) + +cc_library( + name = "snappy-c", + srcs = ["snappy-c.cc"], + hdrs = ["snappy-c.h"], + deps = [":snappy"], +) + +filegroup( + name = "testdata", + srcs = glob(["testdata/*"]), +) + +cc_library( + name = "snappy-test", + testonly = True, + srcs = [ + "snappy-test.cc", + "snappy_test_data.cc", + ], + hdrs = [ + "snappy-test.h", + "snappy_test_data.h", + ], + deps = [":snappy-stubs-internal"], +) + +cc_test( + name = "snappy_benchmark", + srcs = ["snappy_benchmark.cc"], + data = [":testdata"], + deps = [ + ":snappy", + ":snappy-test", + "//third_party/benchmark:benchmark_main", + ], +) + +cc_test( + name = "snappy_unittest", + srcs = [ + "snappy_unittest.cc", + ], + data = [":testdata"], + deps = [ + ":snappy", + ":snappy-test", + "//third_party/googletest:gtest_main", + ], +) + +# Generate a config.h similar to what cmake would produce. +genrule( + name = "config_h", + outs = ["config.h"], + cmd = """cat <$@ +#define HAVE_STDDEF_H 1 +#define HAVE_STDINT_H 1 +#ifdef __has_builtin +# if !defined(HAVE_BUILTIN_EXPECT) && __has_builtin(__builtin_expect) +# define HAVE_BUILTIN_EXPECT 1 +# endif +# if !defined(HAVE_BUILTIN_CTZ) && __has_builtin(__builtin_ctzll) +# define HAVE_BUILTIN_CTZ 1 +# endif +# if !defined(HAVE_BUILTIN_PREFETCH) && __has_builtin(__builtin_prefetech) +# define HAVE_BUILTIN_PREFETCH 1 +# endif +#elif defined(__GNUC__) && (__GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 4) +# ifndef HAVE_BUILTIN_EXPECT +# define HAVE_BUILTIN_EXPECT 1 +# endif +# ifndef HAVE_BUILTIN_CTZ +# define HAVE_BUILTIN_CTZ 1 +# endif +# ifndef HAVE_BUILTIN_PREFETCH +# define HAVE_BUILTIN_PREFETCH 1 +# endif +#endif + +#if defined(_WIN32) && !defined(HAVE_WINDOWS_H) +#define HAVE_WINDOWS_H 1 +#endif + +#ifdef __has_include +# if !defined(HAVE_BYTESWAP_H) && __has_include() +# define HAVE_BYTESWAP_H 1 +# endif +# if !defined(HAVE_UNISTD_H) && __has_include() +# define HAVE_UNISTD_H 1 +# endif +# if !defined(HAVE_SYS_ENDIAN_H) && __has_include() +# define HAVE_SYS_ENDIAN_H 1 +# endif +# if !defined(HAVE_SYS_MMAN_H) && __has_include() +# define HAVE_SYS_MMAN_H 1 +# endif +# if !defined(HAVE_SYS_UIO_H) && __has_include() +# define HAVE_SYS_UIO_H 1 +# endif +# if !defined(HAVE_SYS_TIME_H) && __has_include() +# define HAVE_SYS_TIME_H 1 +# endif +#endif + +#ifndef SNAPPY_IS_BIG_ENDIAN +# ifdef __s390x__ +# define SNAPPY_IS_BIG_ENDIAN 1 +# elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define SNAPPY_IS_BIG_ENDIAN 1 +# endif +#endif +EOF +""", +) + +genrule( + name = "snappy_stubs_public_h", + srcs = ["snappy-stubs-public.h.in"], + outs = ["snappy-stubs-public.h"], + # Assume sys/uio.h is available on non-Windows. + # Set the version numbers. + cmd = ("""sed -e 's/$${HAVE_SYS_UIO_H_01}/!_WIN32/g' \ + -e 's/$${PROJECT_VERSION_MAJOR}/%d/g' \ + -e 's/$${PROJECT_VERSION_MINOR}/%d/g' \ + -e 's/$${PROJECT_VERSION_PATCH}/%d/g' \ + $< >$@""" % SNAPPY_VERSION), +) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3062e2e..644df247 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.1) -project(Snappy VERSION 1.1.10 LANGUAGES C CXX) +project(Snappy VERSION 1.2.1 LANGUAGES C CXX) # C++ standard can be overridden when this is used as a sub-project. if(NOT CMAKE_CXX_STANDARD) @@ -73,6 +73,11 @@ else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") endif(NOT CMAKE_CXX_FLAGS MATCHES "-Werror") endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # Disable sign comparison warnings. Matches upcoming Bazel setup. + if(NOT CMAKE_CXX_FLAGS MATCHES "-Wno-sign-compare") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") + endif(NOT CMAKE_CXX_FLAGS MATCHES "-Wno-sign-compare") + # Disable C++ exceptions. string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions") @@ -141,6 +146,8 @@ endif(SNAPPY_REQUIRE_AVX2) # Used by googletest. check_cxx_compiler_flag(-Wno-missing-field-initializers SNAPPY_HAVE_NO_MISSING_FIELD_INITIALIZERS) +check_cxx_compiler_flag(-Wno-implicit-int-float-conversion + SNAPPY_HAVE_NO_IMPLICIT_INT_FLOAT_CONVERSION) include(CheckCXXSourceCompiles) check_cxx_source_compiles(" @@ -153,6 +160,12 @@ int main() { return __builtin_ctzll(0); }" HAVE_BUILTIN_CTZ) +check_cxx_source_compiles(" +int main() { + __builtin_prefetch(0, 0, 3); + return 0; +}" HAVE_BUILTIN_PREFETCH) + check_cxx_source_compiles(" __attribute__((always_inline)) int zero() { return 0; } @@ -323,6 +336,11 @@ if(SNAPPY_BUILD_TESTS) APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers) endif(SNAPPY_HAVE_NO_MISSING_FIELD_INITIALIZERS) + if(SNAPPY_HAVE_NO_IMPLICIT_INT_FLOAT_CONVERSION) + set_property(TARGET gtest + APPEND PROPERTY COMPILE_OPTIONS -Wno-implicit-int-float-conversion) + endif(SNAPPY_HAVE_NO_IMPLICIT_INT_FLOAT_CONVERSION) + add_executable(snappy_unittest "") target_sources(snappy_unittest PRIVATE diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 00000000..7e60888b --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,27 @@ +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cmake/config.h.in b/cmake/config.h.in index d1de25cf..3510c27e 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -10,6 +10,9 @@ /* Define to 1 if the compiler supports __builtin_expect. */ #cmakedefine01 HAVE_BUILTIN_EXPECT +/* Define to 1 if the compiler supports __builtin_prefetch. */ +#cmakedefine01 HAVE_BUILTIN_PREFETCH + /* Define to 1 if you have a definition for mmap() in . */ #cmakedefine01 HAVE_FUNC_MMAP diff --git a/snappy-internal.h b/snappy-internal.h index 0923f399..ae78247d 100644 --- a/snappy-internal.h +++ b/snappy-internal.h @@ -31,6 +31,8 @@ #ifndef THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_ #define THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_ +#include + #include "snappy-stubs-internal.h" #if SNAPPY_HAVE_SSSE3 @@ -256,6 +258,8 @@ static inline std::pair FindMatchLength(const char* s1, s2 += 8; } } + SNAPPY_PREFETCH(s1 + 64); + SNAPPY_PREFETCH(s2 + 64); // Find out how long the match is. We loop over the data 64 bits at a // time until we find a 64-bit block that doesn't match; then we find @@ -330,6 +334,31 @@ static inline std::pair FindMatchLength(const char* s1, } #endif +static inline size_t FindMatchLengthPlain(const char* s1, const char* s2, + const char* s2_limit) { + // Implementation based on the x86-64 version, above. + assert(s2_limit >= s2); + int matched = 0; + + while (s2 <= s2_limit - 8 && + UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) { + s2 += 8; + matched += 8; + } + if (LittleEndian::IsLittleEndian() && s2 <= s2_limit - 8) { + uint64_t x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched); + int matching_bits = Bits::FindLSBSetNonZero64(x); + matched += matching_bits >> 3; + s2 += matching_bits >> 3; + } else { + while ((s2 < s2_limit) && (s1[matched] == *s2)) { + ++s2; + ++matched; + } + } + return matched; +} + // Lookup tables for decompression code. Give --snappy_dump_decompression_table // to the unit test to recompute char_table. diff --git a/snappy-stubs-internal.h b/snappy-stubs-internal.h index 1548ed7a..526c38b7 100644 --- a/snappy-stubs-internal.h +++ b/snappy-stubs-internal.h @@ -105,6 +105,12 @@ #define SNAPPY_ATTRIBUTE_ALWAYS_INLINE #endif // HAVE_ATTRIBUTE_ALWAYS_INLINE +#if HAVE_BUILTIN_PREFETCH +#define SNAPPY_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 3) +#else +#define SNAPPY_PREFETCH(ptr) (void)(ptr) +#endif + // Stubbed version of ABSL_FLAG. // // In the open source version, flags can only be changed at compile time. diff --git a/snappy.cc b/snappy.cc index d4147185..56b8c82a 100644 --- a/snappy.cc +++ b/snappy.cc @@ -68,18 +68,13 @@ #include #endif -#if defined(__GNUC__) -#define SNAPPY_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 3) -#else -#define SNAPPY_PREFETCH(ptr) (void)(ptr) -#endif - #include #include #include #include #include #include +#include #include #include #include @@ -180,6 +175,22 @@ inline uint16_t* TableEntry(uint16_t* table, uint32_t bytes, uint32_t mask) { (hash & mask)); } +inline uint16_t* TableEntry4ByteMatch(uint16_t* table, uint32_t bytes, + uint32_t mask) { + constexpr uint32_t kMagic = 2654435761U; + const uint32_t hash = (kMagic * bytes) >> (32 - kMaxHashTableBits); + return reinterpret_cast(reinterpret_cast(table) + + (hash & mask)); +} + +inline uint16_t* TableEntry8ByteMatch(uint16_t* table, uint64_t bytes, + uint32_t mask) { + constexpr uint64_t kMagic = 58295818150454627ULL; + const uint32_t hash = (kMagic * bytes) >> (64 - kMaxHashTableBits); + return reinterpret_cast(reinterpret_cast(table) + + (hash & mask)); +} + } // namespace size_t MaxCompressedLength(size_t source_bytes) { @@ -936,6 +947,174 @@ char* CompressFragment(const char* input, size_t input_size, char* op, } } +emit_remainder: + // Emit the remaining bytes as a literal + if (ip < ip_end) { + op = EmitLiteral(op, ip, ip_end - ip); + } + + return op; +} + +char* CompressFragmentDoubleHash(const char* input, size_t input_size, char* op, + uint16_t* table, const int table_size, + uint16_t* table2, const int table_size2) { + (void)table_size2; + assert(table_size == table_size2); + // "ip" is the input pointer, and "op" is the output pointer. + const char* ip = input; + assert(input_size <= kBlockSize); + assert((table_size & (table_size - 1)) == 0); // table must be power of two + const uint32_t mask = 2 * (table_size - 1); + const char* ip_end = input + input_size; + const char* base_ip = ip; + + const size_t kInputMarginBytes = 15; + if (SNAPPY_PREDICT_TRUE(input_size >= kInputMarginBytes)) { + const char* ip_limit = input + input_size - kInputMarginBytes; + + for (;;) { + const char* next_emit = ip++; + uint64_t data = LittleEndian::Load64(ip); + uint32_t skip = 512; + + const char* candidate; + uint32_t candidate_length; + while (true) { + assert(static_cast(data) == LittleEndian::Load32(ip)); + uint16_t* table_entry2 = TableEntry8ByteMatch(table2, data, mask); + uint32_t bytes_between_hash_lookups = skip >> 9; + skip++; + const char* next_ip = ip + bytes_between_hash_lookups; + if (SNAPPY_PREDICT_FALSE(next_ip > ip_limit)) { + ip = next_emit; + goto emit_remainder; + } + candidate = base_ip + *table_entry2; + assert(candidate >= base_ip); + assert(candidate < ip); + + *table_entry2 = ip - base_ip; + if (SNAPPY_PREDICT_FALSE(static_cast(data) == + LittleEndian::Load32(candidate))) { + candidate_length = + FindMatchLengthPlain(candidate + 4, ip + 4, ip_end) + 4; + break; + } + + uint16_t* table_entry = TableEntry4ByteMatch(table, data, mask); + candidate = base_ip + *table_entry; + assert(candidate >= base_ip); + assert(candidate < ip); + + *table_entry = ip - base_ip; + if (SNAPPY_PREDICT_FALSE(static_cast(data) == + LittleEndian::Load32(candidate))) { + candidate_length = + FindMatchLengthPlain(candidate + 4, ip + 4, ip_end) + 4; + table_entry2 = + TableEntry8ByteMatch(table2, LittleEndian::Load64(ip + 1), mask); + auto candidate2 = base_ip + *table_entry2; + size_t candidate_length2 = + FindMatchLengthPlain(candidate2, ip + 1, ip_end); + if (candidate_length2 > candidate_length) { + *table_entry2 = ip - base_ip; + candidate = candidate2; + candidate_length = candidate_length2; + ++ip; + } + break; + } + data = LittleEndian::Load64(next_ip); + ip = next_ip; + } + // Backtrack to the point it matches fully. + while (ip > next_emit && candidate > base_ip && + *(ip - 1) == *(candidate - 1)) { + --ip; + --candidate; + ++candidate_length; + } + *TableEntry8ByteMatch(table2, LittleEndian::Load64(ip + 1), mask) = + ip - base_ip + 1; + *TableEntry8ByteMatch(table2, LittleEndian::Load64(ip + 2), mask) = + ip - base_ip + 2; + *TableEntry4ByteMatch(table, LittleEndian::Load32(ip + 1), mask) = + ip - base_ip + 1; + // Step 2: A 4-byte or 8-byte match has been found. + // We'll later see if more than 4 bytes match. But, prior to the match, + // input bytes [next_emit, ip) are unmatched. Emit them as + // "literal bytes." + assert(next_emit + 16 <= ip_end); + if (ip - next_emit > 0) { + op = EmitLiteral(op, next_emit, + ip - next_emit); + } + // Step 3: Call EmitCopy, and then see if another EmitCopy could + // be our next move. Repeat until we find no match for the + // input immediately after what was consumed by the last EmitCopy call. + // + // If we exit this loop normally then we need to call EmitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can exit + // this loop via goto if we get close to exhausting the input. + do { + // We have a 4-byte match at ip, and no need to emit any + // "literal bytes" prior to ip. + const char* base = ip; + ip += candidate_length; + size_t offset = base - candidate; + if (candidate_length < 12) { + op = + EmitCopy(op, offset, candidate_length); + } else { + op = EmitCopy(op, offset, + candidate_length); + } + if (SNAPPY_PREDICT_FALSE(ip >= ip_limit)) { + goto emit_remainder; + } + // We are now looking for a 4-byte match again. We read + // table[Hash(ip, mask)] for that. To improve compression, + // we also update several previous table entries. + if (ip - base_ip > 7) { + *TableEntry8ByteMatch(table2, LittleEndian::Load64(ip - 7), mask) = + ip - base_ip - 7; + *TableEntry8ByteMatch(table2, LittleEndian::Load64(ip - 4), mask) = + ip - base_ip - 4; + } + *TableEntry8ByteMatch(table2, LittleEndian::Load64(ip - 3), mask) = + ip - base_ip - 3; + *TableEntry8ByteMatch(table2, LittleEndian::Load64(ip - 2), mask) = + ip - base_ip - 2; + *TableEntry4ByteMatch(table, LittleEndian::Load32(ip - 2), mask) = + ip - base_ip - 2; + *TableEntry4ByteMatch(table, LittleEndian::Load32(ip - 1), mask) = + ip - base_ip - 1; + + uint16_t* table_entry = + TableEntry8ByteMatch(table2, LittleEndian::Load64(ip), mask); + candidate = base_ip + *table_entry; + *table_entry = ip - base_ip; + if (LittleEndian::Load32(ip) == LittleEndian::Load32(candidate)) { + candidate_length = + FindMatchLengthPlain(candidate + 4, ip + 4, ip_end) + 4; + continue; + } + table_entry = + TableEntry4ByteMatch(table, LittleEndian::Load32(ip), mask); + candidate = base_ip + *table_entry; + *table_entry = ip - base_ip; + if (LittleEndian::Load32(ip) == LittleEndian::Load32(candidate)) { + candidate_length = + FindMatchLengthPlain(candidate + 4, ip + 4, ip_end) + 4; + continue; + } + break; + } while (true); + } + } + emit_remainder: // Emit the remaining bytes as a literal if (ip < ip_end) { @@ -946,10 +1125,10 @@ char* CompressFragment(const char* input, size_t input_size, char* op, } } // end namespace internal -// Called back at avery compression call to trace parameters and sizes. -static inline void Report(const char *algorithm, size_t compressed_size, - size_t uncompressed_size) { +static inline void Report(int token, const char *algorithm, size_t +compressed_size, size_t uncompressed_size) { // TODO: Switch to [[maybe_unused]] when we can assume C++17. + (void)token; (void)algorithm; (void)compressed_size; (void)uncompressed_size; @@ -1234,16 +1413,21 @@ std::pair DecompressBranchless( assert(tag == ip[-1]); // For literals tag_type = 0, hence we will always obtain 0 from // ExtractLowBytes. For literals offset will thus be kLiteralOffset. - ptrdiff_t len_min_offset = kLengthMinusOffset[tag]; + ptrdiff_t len_minus_offset = kLengthMinusOffset[tag]; + uint32_t next; #if defined(__aarch64__) size_t tag_type = AdvanceToNextTagARMOptimized(&ip, &tag); + // We never need more than 16 bits. Doing a Load16 allows the compiler + // to elide the masking operation in ExtractOffset. + next = LittleEndian::Load16(old_ip); #else size_t tag_type = AdvanceToNextTagX86Optimized(&ip, &tag); + next = LittleEndian::Load32(old_ip); #endif - uint32_t next = LittleEndian::Load32(old_ip); - size_t len = len_min_offset & 0xFF; - len_min_offset -= ExtractOffset(next, tag_type); - if (SNAPPY_PREDICT_FALSE(len_min_offset > 0)) { + size_t len = len_minus_offset & 0xFF; + ptrdiff_t extracted = ExtractOffset(next, tag_type); + ptrdiff_t len_min_offset = len_minus_offset - extracted; + if (SNAPPY_PREDICT_FALSE(len_minus_offset > extracted)) { if (SNAPPY_PREDICT_FALSE(len & 0x80)) { // Exceptional case (long literal or copy 4). // Actually doing the copy here is negatively impacting the main @@ -1290,7 +1474,7 @@ std::pair DecompressBranchless( DeferMemCopy(&deferred_src, &deferred_length, from, len); } } while (ip < ip_limit_min_slop && - (op + deferred_length) < op_limit_min_slop); + static_cast(op + deferred_length) < op_limit_min_slop); exit: ip--; assert(ip <= ip_limit); @@ -1592,7 +1776,8 @@ template static bool InternalUncompressAllTags(SnappyDecompressor* decompressor, Writer* writer, uint32_t compressed_len, uint32_t uncompressed_len) { - Report("snappy_uncompress", compressed_len, uncompressed_len); + int token = 0; + Report(token, "snappy_uncompress", compressed_len, uncompressed_len); writer->SetExpectedLength(uncompressed_len); @@ -1608,6 +1793,12 @@ bool GetUncompressedLength(Source* source, uint32_t* result) { } size_t Compress(Source* reader, Sink* writer) { + return Compress(reader, writer, CompressionOptions{}); +} + +size_t Compress(Source* reader, Sink* writer, CompressionOptions options) { + assert(options.level == 1 || options.level == 2); + int token = 0; size_t written = 0; size_t N = reader->Available(); const size_t uncompressed_size = N; @@ -1654,17 +1845,23 @@ size_t Compress(Source* reader, Sink* writer) { uint16_t* table = wmem.GetHashTable(num_to_read, &table_size); // Compress input_fragment and append to dest - const int max_output = MaxCompressedLength(num_to_read); - - // Need a scratch buffer for the output, in case the byte sink doesn't - // have room for us directly. + int max_output = MaxCompressedLength(num_to_read); // Since we encode kBlockSize regions followed by a region // which is <= kBlockSize in length, a previously allocated // scratch_output[] region is big enough for this iteration. + // Need a scratch buffer for the output, in case the byte sink doesn't + // have room for us directly. char* dest = writer->GetAppendBuffer(max_output, wmem.GetScratchOutput()); - char* end = internal::CompressFragment(fragment, fragment_size, dest, table, - table_size); + char* end = nullptr; + if (options.level == 1) { + end = internal::CompressFragment(fragment, fragment_size, dest, table, + table_size); + } else if (options.level == 2) { + end = internal::CompressFragmentDoubleHash( + fragment, fragment_size, dest, table, table_size >> 1, + table + (table_size >> 1), table_size >> 1); + } writer->Append(dest, end - dest); written += (end - dest); @@ -1672,8 +1869,7 @@ size_t Compress(Source* reader, Sink* writer) { reader->Skip(pending_advance); } - Report("snappy_compress", written, uncompressed_size); - + Report(token, "snappy_compress", written, uncompressed_size); return written; } @@ -1696,16 +1892,16 @@ class SnappyIOVecReader : public Source { if (total_size > 0 && curr_size_remaining_ == 0) Advance(); } - ~SnappyIOVecReader() = default; + ~SnappyIOVecReader() override = default; - size_t Available() const { return total_size_remaining_; } + size_t Available() const override { return total_size_remaining_; } - const char* Peek(size_t* len) { + const char* Peek(size_t* len) override { *len = curr_size_remaining_; return curr_pos_; } - void Skip(size_t n) { + void Skip(size_t n) override { while (n >= curr_size_remaining_ && n > 0) { n -= curr_size_remaining_; Advance(); @@ -2108,9 +2304,15 @@ bool IsValidCompressed(Source* compressed) { void RawCompress(const char* input, size_t input_length, char* compressed, size_t* compressed_length) { + RawCompress(input, input_length, compressed, compressed_length, + CompressionOptions{}); +} + +void RawCompress(const char* input, size_t input_length, char* compressed, + size_t* compressed_length, CompressionOptions options) { ByteArraySource reader(input, input_length); UncheckedByteArraySink writer(compressed); - Compress(&reader, &writer); + Compress(&reader, &writer, options); // Compute how many bytes were added *compressed_length = (writer.CurrentDestination() - compressed); @@ -2118,9 +2320,16 @@ void RawCompress(const char* input, size_t input_length, char* compressed, void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, char* compressed, size_t* compressed_length) { + RawCompressFromIOVec(iov, uncompressed_length, compressed, compressed_length, + CompressionOptions{}); +} + +void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, + char* compressed, size_t* compressed_length, + CompressionOptions options) { SnappyIOVecReader reader(iov, uncompressed_length); UncheckedByteArraySink writer(compressed); - Compress(&reader, &writer); + Compress(&reader, &writer, options); // Compute how many bytes were added. *compressed_length = writer.CurrentDestination() - compressed; @@ -2128,18 +2337,28 @@ void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, size_t Compress(const char* input, size_t input_length, std::string* compressed) { + return Compress(input, input_length, compressed, CompressionOptions{}); +} + +size_t Compress(const char* input, size_t input_length, std::string* compressed, + CompressionOptions options) { // Pre-grow the buffer to the max length of the compressed output STLStringResizeUninitialized(compressed, MaxCompressedLength(input_length)); size_t compressed_length; RawCompress(input, input_length, string_as_array(compressed), - &compressed_length); + &compressed_length, options); compressed->erase(compressed_length); return compressed_length; } size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, std::string* compressed) { + return CompressFromIOVec(iov, iov_cnt, compressed, CompressionOptions{}); +} + +size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, + std::string* compressed, CompressionOptions options) { // Compute the number of bytes to be compressed. size_t uncompressed_length = 0; for (size_t i = 0; i < iov_cnt; ++i) { @@ -2152,7 +2371,7 @@ size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, size_t compressed_length; RawCompressFromIOVec(iov, uncompressed_length, string_as_array(compressed), - &compressed_length); + &compressed_length, options); compressed->erase(compressed_length); return compressed_length; } @@ -2342,7 +2561,6 @@ bool SnappyScatteredWriter::SlowAppendFromSelf(size_t offset, class SnappySinkAllocator { public: explicit SnappySinkAllocator(Sink* dest) : dest_(dest) {} - ~SnappySinkAllocator() {} char* Allocate(int size) { Datablock block(new char[size], size); diff --git a/snappy.h b/snappy.h index e12b658e..2f1b8020 100644 --- a/snappy.h +++ b/snappy.h @@ -50,13 +50,38 @@ namespace snappy { class Source; class Sink; + struct CompressionOptions { + // Compression level. + // Level 1 is the fastest + // Level 2 is a little slower but provides better compression. Level 2 is + // **EXPERIMENTAL** for the time being. It might happen that we decide to + // fall back to level 1 in the future. + // Levels 3+ are currently not supported. We plan to support levels up to + // 9 in the future. + // If you played with other compression algorithms, level 1 is equivalent to + // fast mode (level 1) of LZ4, level 2 is equivalent to LZ4's level 2 mode + // and compresses somewhere around zstd:-3 and zstd:-2 but generally with + // faster decompression speeds than snappy:1 and zstd:-3. + int level = DefaultCompressionLevel(); + + constexpr CompressionOptions() = default; + constexpr CompressionOptions(int compression_level) + : level(compression_level) {} + static constexpr int MinCompressionLevel() { return 1; } + static constexpr int MaxCompressionLevel() { return 2; } + static constexpr int DefaultCompressionLevel() { return 1; } + }; + // ------------------------------------------------------------------------ // Generic compression/decompression routines. // ------------------------------------------------------------------------ - // Compress the bytes read from "*source" and append to "*sink". Return the + // Compress the bytes read from "*reader" and append to "*writer". Return the // number of bytes written. - size_t Compress(Source* source, Sink* sink); + // First version is to preserve ABI. + size_t Compress(Source* reader, Sink* writer); + size_t Compress(Source* reader, Sink* writer, + CompressionOptions options); // Find the uncompressed length of the given stream, as given by the header. // Note that the true length could deviate from this; the stream could e.g. @@ -75,15 +100,22 @@ namespace snappy { // Original contents of *compressed are lost. // // REQUIRES: "input[]" is not an alias of "*compressed". + // First version is to preserve ABI. size_t Compress(const char* input, size_t input_length, std::string* compressed); + size_t Compress(const char* input, size_t input_length, + std::string* compressed, CompressionOptions options); // Same as `Compress` above but taking an `iovec` array as input. Note that // this function preprocesses the inputs to compute the sum of // `iov[0..iov_cnt-1].iov_len` before reading. To avoid this, use // `RawCompressFromIOVec` below. + // First version is to preserve ABI. size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, std::string* compressed); + size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, + std::string* compressed, + CompressionOptions options); // Decompresses "compressed[0..compressed_length-1]" to "*uncompressed". // Original contents of "*uncompressed" are lost. @@ -126,16 +158,19 @@ namespace snappy { // RawCompress(input, input_length, output, &output_length); // ... Process(output, output_length) ... // delete [] output; - void RawCompress(const char* input, - size_t input_length, - char* compressed, + void RawCompress(const char* input, size_t input_length, char* compressed, size_t* compressed_length); + void RawCompress(const char* input, size_t input_length, char* compressed, + size_t* compressed_length, CompressionOptions options); // Same as `RawCompress` above but taking an `iovec` array as input. Note that // `uncompressed_length` is the total number of bytes to be read from the // elements of `iov` (_not_ the number of elements in `iov`). void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, char* compressed, size_t* compressed_length); + void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, + char* compressed, size_t* compressed_length, + CompressionOptions options); // Given data in "compressed[0..compressed_length-1]" generated by // calling the Snappy::Compress routine, this routine @@ -215,7 +250,7 @@ namespace snappy { static constexpr int kMinHashTableBits = 8; static constexpr size_t kMinHashTableSize = 1 << kMinHashTableBits; - static constexpr int kMaxHashTableBits = 14; + static constexpr int kMaxHashTableBits = 15; static constexpr size_t kMaxHashTableSize = 1 << kMaxHashTableBits; } // end namespace snappy diff --git a/snappy_benchmark.cc b/snappy_benchmark.cc index 28570dd2..d6e35d38 100644 --- a/snappy_benchmark.cc +++ b/snappy_benchmark.cc @@ -31,12 +31,10 @@ #include #include -#include "snappy-test.h" - #include "benchmark/benchmark.h" - #include "snappy-internal.h" #include "snappy-sinksource.h" +#include "snappy-test.h" #include "snappy.h" #include "snappy_test_data.h" @@ -44,6 +42,15 @@ namespace snappy { namespace { +void FilesAndLevels(benchmark::internal::Benchmark* benchmark) { + for (int i = 0; i < ARRAYSIZE(kTestDataFiles); ++i) { + for (int level = snappy::CompressionOptions::MinCompressionLevel(); + level <= snappy::CompressionOptions::MaxCompressionLevel(); ++level) { + benchmark->ArgPair(i, level); + } + } +} + void BM_UFlat(benchmark::State& state) { // Pick file to process based on state.range(0). int file_index = state.range(0); @@ -55,7 +62,9 @@ void BM_UFlat(benchmark::State& state) { kTestDataFiles[file_index].size_limit); std::string zcontents; - snappy::Compress(contents.data(), contents.size(), &zcontents); + snappy::Compress( + contents.data(), contents.size(), &zcontents, + snappy::CompressionOptions{/*level=*/static_cast(state.range(1))}); char* dst = new char[contents.size()]; for (auto s : state) { @@ -68,7 +77,7 @@ void BM_UFlat(benchmark::State& state) { delete[] dst; } -BENCHMARK(BM_UFlat)->DenseRange(0, ARRAYSIZE(kTestDataFiles) - 1); +BENCHMARK(BM_UFlat)->Apply(FilesAndLevels); struct SourceFiles { SourceFiles() { @@ -119,7 +128,9 @@ void BM_UValidate(benchmark::State& state) { kTestDataFiles[file_index].size_limit); std::string zcontents; - snappy::Compress(contents.data(), contents.size(), &zcontents); + snappy::Compress( + contents.data(), contents.size(), &zcontents, + snappy::CompressionOptions{/*level=*/static_cast(state.range(1))}); for (auto s : state) { CHECK(snappy::IsValidCompressedBuffer(zcontents.data(), zcontents.size())); @@ -128,7 +139,7 @@ void BM_UValidate(benchmark::State& state) { static_cast(contents.size())); state.SetLabel(kTestDataFiles[file_index].label); } -BENCHMARK(BM_UValidate)->DenseRange(0, ARRAYSIZE(kTestDataFiles) - 1); +BENCHMARK(BM_UValidate)->Apply(FilesAndLevels); void BM_UValidateMedley(benchmark::State& state) { static const SourceFiles* const source = new SourceFiles(); @@ -152,6 +163,7 @@ BENCHMARK(BM_UValidateMedley); void BM_UIOVecSource(benchmark::State& state) { // Pick file to process based on state.range(0). int file_index = state.range(0); + int level = state.range(1); CHECK_GE(file_index, 0); CHECK_LT(file_index, ARRAYSIZE(kTestDataFiles)); @@ -180,7 +192,8 @@ void BM_UIOVecSource(benchmark::State& state) { char* dst = new char[snappy::MaxCompressedLength(contents.size())]; size_t zsize = 0; for (auto s : state) { - snappy::RawCompressFromIOVec(iov, contents.size(), dst, &zsize); + snappy::RawCompressFromIOVec(iov, contents.size(), dst, &zsize, + snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(iov); } state.SetBytesProcessed(static_cast(state.iterations()) * @@ -195,7 +208,7 @@ void BM_UIOVecSource(benchmark::State& state) { delete[] dst; } -BENCHMARK(BM_UIOVecSource)->DenseRange(0, ARRAYSIZE(kTestDataFiles) - 1); +BENCHMARK(BM_UIOVecSource)->Apply(FilesAndLevels); void BM_UIOVecSink(benchmark::State& state) { // Pick file to process based on state.range(0). @@ -213,7 +226,7 @@ void BM_UIOVecSink(benchmark::State& state) { // Uncompress into an iovec containing ten entries. const int kNumEntries = 10; struct iovec iov[kNumEntries]; - char *dst = new char[contents.size()]; + char* dst = new char[contents.size()]; size_t used_so_far = 0; for (int i = 0; i < kNumEntries; ++i) { iov[i].iov_base = dst + used_so_far; @@ -254,7 +267,9 @@ void BM_UFlatSink(benchmark::State& state) { kTestDataFiles[file_index].size_limit); std::string zcontents; - snappy::Compress(contents.data(), contents.size(), &zcontents); + snappy::Compress( + contents.data(), contents.size(), &zcontents, + snappy::CompressionOptions{/*level=*/static_cast(state.range(1))}); char* dst = new char[contents.size()]; for (auto s : state) { @@ -273,11 +288,12 @@ void BM_UFlatSink(benchmark::State& state) { delete[] dst; } -BENCHMARK(BM_UFlatSink)->DenseRange(0, ARRAYSIZE(kTestDataFiles) - 1); +BENCHMARK(BM_UFlatSink)->Apply(FilesAndLevels); void BM_ZFlat(benchmark::State& state) { // Pick file to process based on state.range(0). int file_index = state.range(0); + int level = state.range(1); CHECK_GE(file_index, 0); CHECK_LT(file_index, ARRAYSIZE(kTestDataFiles)); @@ -288,7 +304,8 @@ void BM_ZFlat(benchmark::State& state) { size_t zsize = 0; for (auto s : state) { - snappy::RawCompress(contents.data(), contents.size(), dst, &zsize); + snappy::RawCompress(contents.data(), contents.size(), dst, &zsize, + snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(dst); } state.SetBytesProcessed(static_cast(state.iterations()) * @@ -302,10 +319,12 @@ void BM_ZFlat(benchmark::State& state) { zsize); delete[] dst; } -BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(kTestDataFiles) - 1); + +BENCHMARK(BM_ZFlat)->Apply(FilesAndLevels); void BM_ZFlatAll(benchmark::State& state) { const int num_files = ARRAYSIZE(kTestDataFiles); + int level = state.range(0); std::vector contents(num_files); std::vector dst(num_files); @@ -322,7 +341,7 @@ void BM_ZFlatAll(benchmark::State& state) { for (auto s : state) { for (int i = 0; i < num_files; ++i) { snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], - &zsize); + &zsize, snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(dst); } } @@ -335,10 +354,11 @@ void BM_ZFlatAll(benchmark::State& state) { } state.SetLabel(StrFormat("%d kTestDataFiles", num_files)); } -BENCHMARK(BM_ZFlatAll); +BENCHMARK(BM_ZFlatAll)->DenseRange(1, 2); void BM_ZFlatIncreasingTableSize(benchmark::State& state) { CHECK_GT(ARRAYSIZE(kTestDataFiles), 0); + int level = state.range(0); const std::string base_content = ReadTestDataFile( kTestDataFiles[0].filename, kTestDataFiles[0].size_limit); @@ -358,7 +378,7 @@ void BM_ZFlatIncreasingTableSize(benchmark::State& state) { for (auto s : state) { for (size_t i = 0; i < contents.size(); ++i) { snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], - &zsize); + &zsize, snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(dst); } } @@ -371,7 +391,7 @@ void BM_ZFlatIncreasingTableSize(benchmark::State& state) { } state.SetLabel(StrFormat("%d tables", contents.size())); } -BENCHMARK(BM_ZFlatIncreasingTableSize); +BENCHMARK(BM_ZFlatIncreasingTableSize)->DenseRange(1, 2); } // namespace diff --git a/snappy_compress_fuzzer.cc b/snappy_compress_fuzzer.cc index 1d4429a8..93254a28 100644 --- a/snappy_compress_fuzzer.cc +++ b/snappy_compress_fuzzer.cc @@ -39,22 +39,26 @@ // Entry point for LibFuzzer. extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { std::string input(reinterpret_cast(data), size); + for (int level = snappy::CompressionOptions::MinCompressionLevel(); + level <= snappy::CompressionOptions::MaxCompressionLevel(); ++level) { + std::string compressed; + size_t compressed_size = + snappy::Compress(input.data(), input.size(), &compressed, + snappy::CompressionOptions{/*level=*/level}); - std::string compressed; - size_t compressed_size = - snappy::Compress(input.data(), input.size(), &compressed); + (void)compressed_size; // Variable only used in debug builds. + assert(compressed_size == compressed.size()); + assert(compressed.size() <= snappy::MaxCompressedLength(input.size())); + assert( + snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); - (void)compressed_size; // Variable only used in debug builds. - assert(compressed_size == compressed.size()); - assert(compressed.size() <= snappy::MaxCompressedLength(input.size())); - assert(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + std::string uncompressed_after_compress; + bool uncompress_succeeded = snappy::Uncompress( + compressed.data(), compressed.size(), &uncompressed_after_compress); - std::string uncompressed_after_compress; - bool uncompress_succeeded = snappy::Uncompress( - compressed.data(), compressed.size(), &uncompressed_after_compress); - - (void)uncompress_succeeded; // Variable only used in debug builds. - assert(uncompress_succeeded); - assert(input == uncompressed_after_compress); + (void)uncompress_succeeded; // Variable only used in debug builds. + assert(uncompress_succeeded); + assert(input == uncompressed_after_compress); + } return 0; } diff --git a/third_party/benchmark b/third_party/benchmark index bf585a27..d572f477 160000 --- a/third_party/benchmark +++ b/third_party/benchmark @@ -1 +1 @@ -Subproject commit bf585a2789e30585b4e3ce6baf11ef2750b54677 +Subproject commit d572f4777349d43653b21d6c2fc63020ab326db2 diff --git a/third_party/googletest b/third_party/googletest index 18f8200e..b796f7d4 160000 --- a/third_party/googletest +++ b/third_party/googletest @@ -1 +1 @@ -Subproject commit 18f8200e3079b0e54fa00cb7ac55d4c39dcf6da6 +Subproject commit b796f7d44681514f58a683a3a71ff17c94edb0c1 From 68534d0a3796a2b97ea6d024889f54f4517d8c4e Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 4 Jul 2024 10:47:29 +0200 Subject: [PATCH 9/9] Update snappy version in documentation --- doc/information.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/information.rst b/doc/information.rst index 4f8c470a..9b6f28dc 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -95,7 +95,7 @@ HDF5 compression filters and compression libraries sources were obtained from: Sources of compression libraries shared accross multiple filters were obtained from: * `LZ4 v1.9.4 `_ -* `Snappy v1.1.10 `_ +* `Snappy v1.2.1 `_ * `ZStd v1.5.5 `_ * `ZLib v1.2.13 `_