diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2c069f83..8d73fb0b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,12 +2,19 @@ name: h5z-zfp # Controls when the action will run. #Triggers the workflow on push or pull requests. + +# TODO: REMOVE feat-cmake_tests before merging on: push: - branches: [ master ] + branches: [ master ] pull_request: branches: [ master ] +# Using concurrency to cancel any in-progress job or run +concurrency: + group: ${{ github.workflow }}-${{ github.sha || github.event.pull_request.number }} + cancel-in-progress: true + # A workflow run is made up of one or more jobs that # can run sequentially or in parallel jobs: @@ -32,9 +39,10 @@ jobs: - name: Install Dependencies (Linux) if: matrix.os == 'ubuntu-latest' run: | + sudo apt-get update sudo apt-get install -qq libhdf5-dev sudo apt-get install -qq hdf5-tools - echo "HDF5_HOME=/usr/include/hdf5/serial,/usr/lib/x86_64-linux-gnu/hdf5/serial,/usr/bin" >> $GITHUB_ENV + echo "HDF5_HOME=/usr/include/hdf5/serial,/usr/lib/x86_64-linux-gnu/hdf5/serial,/usr/bin" >> $GITHUB_ENV # Set env vars echo "CC=gcc" >> $GITHUB_ENV echo "FC=gfortran" >> $GITHUB_ENV @@ -42,7 +50,7 @@ jobs: # Checks-out the repository under $GITHUB_WORKSPACE so the job can access it. - name: Get Sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 ################################## # INSTALL ZFP @@ -71,11 +79,12 @@ jobs: make check shell: bash - - name: build (cmake) + - name: build (cmake) and test run: | mkdir build; cd build - cmake -D FORTRAN_INTERFACE=ON -D CMAKE_BUILD_TYPE=Release .. - make + cmake -D FORTRAN_INTERFACE=ON -D CMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON .. + sudo cmake --build . --target install + ctest -C Release . shell: bash @@ -87,7 +96,7 @@ jobs: runs-on: windows-2019 steps: - name: checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: install prerequisites run: | choco install \ @@ -98,10 +107,10 @@ jobs: - name: download HDF5 for Windows run: | wget --quiet \ - https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.1/bin/windows/hdf5-1.12.1-Std-win10_64-vs16.zip - unzip hdf5-1.12.1-Std-win10_64-vs16.zip + https://support.hdfgroup.org//ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/windows/hdf5-1.14.1-2-Std-win10_64-vs16-clang.zip + unzip hdf5-1.14.1-2-Std-win10_64-vs16-clang.zip - name: Install HDF5 for Windows - run: msiexec /i "hdf\HDF5-1.12.1-win64.msi" /qn + run: msiexec /i "hdf\HDF5-1.14.1-win64.msi" /qn shell: cmd - name: Download and install ZFP run: | @@ -111,23 +120,33 @@ jobs: mkdir -p build cd build cmake .. \ - -DCMAKE_BUILD_TYPE=Release \ + -G "Visual Studio 16 2019" -A "x64" \ + -DCMAKE_CONFIGURATION_TYPES=Release \ -DBUILD_UTILITIES=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_CFP=ON \ -DZFP_BIT_STREAM_WORD_SIZE=8 - cmake --build . -j 3 - cmake --build . --target install - - name: update PATH + cmake --build . --config Release -j 3 + cmake --build . --config Release --target install + - name: Set build and test variables run: | - echo "C:/Program Files/HDF_Group/HDF5/1.12.1/bin" >> $GITHUB_PATH - - name: configure cmake + echo "C:/Program Files/H5Z_ZFP/plugin" >> $GITHUB_PATH + echo "C:/Program Files/ZFP/bin" >> $GITHUB_PATH + echo "C:/Program Files/ZFP/lib" >> $GITHUB_PATH + echo "C:/Program Files/HDF_Group/HDF5/1.14.1/bin" >> $GITHUB_PATH + echo "HDF5_PLUGIN_PATH=C:/Program Files/H5Z_ZFP/plugin" >> $GITHUB_ENV + echo "HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.14.1" >> $GITHUB_ENV + echo "HDF5_ROOT=C:/Program Files/HDF_Group/HDF5/1.14.1" >> $GITHUB_ENV + echo "ZFP_DIR=C:/Program Files/ZFP" >> $GITHUB_ENV + - name: Build H5Z-ZFP (CMake) and test run: | mkdir build cd build cmake .. \ - -DCMAKE_BUILD_TYPE=Release \ + -G "Visual Studio 16 2019" -A "x64" \ + -DCMAKE_CONFIGURATION_TYPES=Release \ + -DBUILD_TESTING=ON \ -DFORTRAN_INTERFACE=OFF \ -TClangCL - cmake --build . -j 3 - cmake --build . --target install + cmake --build . --config Release --target install + ctest -C Release . diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..f3aa1f5a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index de16e5a4..ccf69a08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,24 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) "separate from the source directory") endif () +if (POLICY CMP0074) + # find_package() uses _ROOT variables. + cmake_policy (SET CMP0074 NEW) +endif () + +if (POLICY CMP0083) + # To control generation of Position Independent Executable (PIE) or not, + # some flags are required at link time. + cmake_policy (SET CMP0083 NEW) +endif () + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + #------------------------------------------------------------------------------# # Parse version number from H5Zzfp_version.h #------------------------------------------------------------------------------# @@ -20,6 +38,11 @@ set(H5Z_ZFP_VERSION "${H5Z_ZFP_VERSION_MAJOR}.${H5Z_ZFP_VERSION_MINOR}.${H5Z_ZFP project(H5Z_ZFP VERSION ${H5Z_ZFP_VERSION} LANGUAGES C) +# setup common output folders +include (cmake/HDFMacros.cmake) +SET_HDF_BUILD_TYPE() +DEFAULT_FOLDERS() + #------------------------------------------------------------------------------# # Compile options. #------------------------------------------------------------------------------# @@ -27,10 +50,11 @@ option(FORTRAN_INTERFACE "Enable the Fortran interface" ON) if (FORTRAN_INTERFACE) enable_language(Fortran) endif () - #------------------------------------------------------------------------------# # Some boilerplate to setup nice output directories #------------------------------------------------------------------------------# +set(CMAKE_INSTALL_BINDIR bin) +set(CMAKE_INSTALL_LIBDIR lib) set(CMAKE_INSTALL_INCLUDEDIR include) set(CMAKE_INSTALL_CMAKEDIR lib/cmake/h5z_zfp) @@ -38,20 +62,40 @@ set(CMAKE_INSTALL_CMAKEDIR lib/cmake/h5z_zfp) # Required packages #------------------------------------------------------------------------------# # Find HDF5, relies on HDF5_DIR or HDF5_ROOT being set in environment. -set(COMPONENTS C) -if (FORTRAN_INTERFACE) - list(APPEND COMPONENTS Fortran) +HDF5_SUPPORT () +if (NOT HDF5_FOUND) + message (FATAL_ERROR "HDF5 is Required for h5z_zfp") endif () -find_package(HDF5 REQUIRED COMPONENTS ${COMPONENTS}) + +# Find MPI depending on if HDF5 needs MPI. +if ((HDF5_IS_PARALLEL OR HDF5_ENABLE_PARALLEL) AND ("${HDF5_VERSION}" VERSION_EQUAL "1.14.0")) + find_package(MPI REQUIRED COMPONENTS C) +endif() # Find ZFP, relies on ZFP_DIR or ZFP_ROOT being set in environment. find_package(ZFP REQUIRED CONFIG) +if (NOT ZFP_FOUND) + message (FATAL_ERROR "ZFP is Required for h5z_zfp") +endif () #------------------------------------------------------------------------------# # Add source #------------------------------------------------------------------------------# add_subdirectory(${CMAKE_SOURCE_DIR}/src) +#----------------------------------------------------------------------------- +# Dashboard and Testing Settings +#----------------------------------------------------------------------------- +option (BUILD_TESTING "Build h5z-zfp Unit Testing" OFF) +if (BUILD_TESTING) + set (DART_TESTING_TIMEOUT 1200 CACHE STRING + "Timeout in seconds for each test (default 1200=20minutes)") + enable_testing () + include (CTest) + + add_subdirectory(${CMAKE_SOURCE_DIR}/test) +endif () + #------------------------------------------------------------------------------# # Packaging #------------------------------------------------------------------------------# diff --git a/Makefile b/Makefile index 32b59690..a29bb266 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,16 @@ -include config.make +# Include config.make only if we're not looking for help +ifeq ($(findstring help, $(strip $(MAKECMDGOALS))),) + ifeq ($(findstring tools, $(strip $(MAKECMDGOALS))),) + ifeq ($(findstring clean, $(strip $(MAKECMDGOALS))),) + ifeq ($(findstring dist, $(strip $(MAKECMDGOALS))),) + include config.make + endif + endif + endif +endif + +# Get version string from H5Z-ZFP +H5Z_ZFP_VERSINFO := $(shell grep '^\#define H5Z_FILTER_ZFP_VERSION_[MP]' src/H5Zzfp_version.h | cut -d' ' -f3 | tr '\n' '.' | cut -d'.' -f-3 2>/dev/null) .PHONY: help all clean dist install @@ -6,7 +18,7 @@ help: @echo "" @echo "" @echo "" - @echo " This is H5Z-ZFP version $(H5Z_ZFP_VERSINFO)." + @echo " This is H5Z-ZFP version $(H5Z_ZFP_VERSINFO)." @echo "See http://h5z-zfp.readthedocs.io/en/latest/ file for more info." @echo "" @echo "Typical make command is..." @@ -14,25 +26,30 @@ help: @echo " make CC= HDF5_HOME= ZFP_HOME= PREFIX= all" @echo "" @echo "where is a dir whose children are include/lib/bin subdirs." - @echo "HDF5_HOME can also be specified by the HDF5 include directory," - @echo "library directory and bin directory separated by commas, i.e. HDF5_HOME=INC,LIB,BIN" + @echo "HDF5_HOME can also be set using an INC,LIB,BIN triplet specifying" + @echo "HDF5 include, library and binary dirs separated by commas." @echo "Standard make variables (e.g. CFLAGS, LD, etc.) can be set as usual." @echo "Optionally, add FC= to include Fortran support and tests." @echo "" @echo "Available make targets are..." - @echo " all - build everything" - @echo " check - all + run tests" - @echo " install - install compiled components" - @echo " clean - clean away all derived targets" - @echo " dist - create distribution tarfile" + @echo " all - build everything needed for H5Z-ZFP plugin/lib" + @echo " check - all + run tests" + @echo " tools - build tools (currently just print_h5repack_farg)" + @echo " install - install plugin/lib" + @echo " clean - clean away all derived targets" + @echo " dist - create distribution tarfile" + @echo " help - this help message" all: cd src; $(MAKE) $(MAKEVARS) $@ -check: all install +check: all cd test; $(MAKE) $(MAKEVARS) $@ -install: +tools: + cd test; $(MAKE) $(MAKEVARS) print_h5repack_farg + +install: all cd src; $(MAKE) $(MAKEVARS) $@ clean: diff --git a/README.md b/README.md index 2b9399c6..f8488053 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,16 @@ compression plugin for the HDF5 library using ZFP compression. For information about ZFP compression and the BSD-Licensed ZFP library, see... -- https://computation.llnl.gov/projects/floating-point-compression +- https://computing.llnl.gov/projects/zfp - https://github.com/LLNL/zfp For information about HDF5 filter plugins, see... - https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters -This H5Z-ZFP plugin supports ZFP versions 0.5.0 through 0.5.5. +This H5Z-ZFP plugin supports ZFP versions 0.5.0 and newer. -This plugin uses the [*registered*](https://support.hdfgroup.org/services/filters.html#zfp) +This plugin uses the [*registered*](https://portal.hdfgroup.org/display/support/Filters#Filters-32013) HDF5 plugin filter id 32013 The HDF5 filter plugin code here is also part of the Silo library. @@ -28,7 +28,7 @@ package due to the likely broad appeal and utility of the ZFP compression library. This plugin supports all modes of the ZFP compression library, *rate*, -*accuracy*, *precision*, *expert* and *lossless*. It supports 1, 2, 3 and +*accuracy*, *precision*, *expert* and *reversible*. It supports 1, 2, 3 and 4 dimensional datasets (for ZFP version 0.5.5 and newer) of single and double precision integer and floating point data. It can be applied to HDF5 datasets of more than 3 dimensions (or 4 dimensions for ZFP versions 0.5.5 and newer) diff --git a/cmake/HDFMacros.cmake b/cmake/HDFMacros.cmake new file mode 100644 index 00000000..5aaacd3d --- /dev/null +++ b/cmake/HDFMacros.cmake @@ -0,0 +1,249 @@ +#------------------------------------------------------------------------------- +macro (DEFAULT_FOLDERS) + #----------------------------------------------------------------------------- + # Setup output Directories + #----------------------------------------------------------------------------- + if (NOT ${H5ZZFP_PACKAGE_NAME}_EXTERNALLY_CONFIGURED) + set (CMAKE_RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." + ) + set (CMAKE_LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" + ) + set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." + ) + set (CMAKE_Fortran_MODULE_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all fortran modules." + ) + if(_isMultiConfig) + set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_BUILD_TYPE}) + set (CMAKE_PDB_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all pdb files." + ) + else () + set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + endif () + else () + # if we are externally configured, but the project uses old cmake scripts + # this may not be set + if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (HDF5_SUPPORT) + if (NOT H5Z-ZFP_HDF5_HEADER) + set (FIND_HDF_COMPONENTS C shared) + message(STATUS "HDF5 FORTRAN_INTERFACE ${FORTRAN_INTERFACE}" ) + if (FORTRAN_INTERFACE) + set (FIND_HDF_COMPONENTS ${FIND_HDF_COMPONENTS} Fortran) + endif () + message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") + set (SEARCH_PACKAGE_NAME "HDF5") + + find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) + message (STATUS "HDF5 C libs:${HDF5_FOUND} static:${HDF5_static_C_FOUND} and shared:${HDF5_shared_C_FOUND}") + message (STATUS "HDF5 Fortran libs: static:${HDF5_static_Fortran_FOUND} and shared:${HDF5_shared_Fortran_FOUND}") + if (HDF5_FOUND) + if (HDF5_shared_C_FOUND) + if (NOT TARGET ${HDF5_NAMESPACE}h5dump-shared) + add_executable (${HDF5_NAMESPACE}h5dump-shared IMPORTED) + endif () + set (HDF5_DUMP_EXECUTABLE $) + + if (NOT TARGET ${HDF5_NAMESPACE}h5diff-shared) + add_executable (${HDF5_NAMESPACE}h5diff-shared IMPORTED) + endif () + set (HDF5_DIFF_EXECUTABLE $) + + if (NOT TARGET ${HDF5_NAMESPACE}h5repack-shared) + add_executable (${HDF5_NAMESPACE}h5repack-shared IMPORTED) + endif () + set (HDF5_REPACK_EXECUTABLE $) + message (STATUS "HDF5 shared Tools found - ${HDF5_DUMP_EXECUTABLE}") + else () + if (NOT TARGET ${HDF5_NAMESPACE}h5dump) + add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) + endif () + set (HDF5_DUMP_EXECUTABLE $) + + if (NOT TARGET ${HDF5_NAMESPACE}h5diff) + add_executable (${HDF5_NAMESPACE}h5diff IMPORTED) + endif () + set (HDF5_DIFF_EXECUTABLE $) + + if (NOT TARGET ${HDF5_NAMESPACE}h5repack) + add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) + endif () + set (HDF5_REPACK_EXECUTABLE $) + message (STATUS "HDF5 static Tools found - ${HDF5_DUMP_EXECUTABLE}") + endif() + + if (NOT HDF5_static_C_FOUND AND NOT HDF5_shared_C_FOUND) + #find library from non-dual-binary package + set (FIND_HDF_COMPONENTS C) + if (FORTRAN_INTERFACE) + set (FIND_HDF_COMPONENTS ${FIND_HDF_COMPONENTS} Fortran) + endif () + message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") + + find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) + message (STATUS "HDF5 libs:${HDF5_FOUND} C:${HDF5_C_FOUND} Fortran:${HDF5_Fortran_FOUND}") + if (HDF5_BUILD_SHARED_LIBS) + add_definitions (-DH5_BUILT_AS_DYNAMIC_LIB) + else () + add_definitions (-DH5_BUILT_AS_STATIC_LIB) + endif () + if (FORTRAN_INTERFACE AND ${HDF5_BUILD_FORTRAN}) + if (HDF5_shared_Fortran_FOUND) + set (HDF5_FORTRAN_INCLUDE_DIRS ${HDF5_INCLUDE_DIR_FORTRAN}) + set (HDF5_FORTRAN_LIBRARIES ${HDF5_FORTRAN_SHARED_LIBRARY}) + else () + set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) + message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") + endif () + endif () + if (WIN32) + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dumpdll") + set_property (TARGET ${HDF5_NAMESPACE}h5diff PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5diffdll") + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repackdll") + endif () + set (HDF5_DUMP_EXECUTABLE $) + set (HDF5_DIFF_EXECUTABLE $) + set (HDF5_REPACK_EXECUTABLE $) + message (STATUS "HDF5 windows Tools found - ${HDF5_DUMP_EXECUTABLE}") + else () + if (HDF5_shared_C_FOUND) + set (HDF5_LIBRARIES ${HDF5_C_SHARED_LIBRARY}) + set (HDF5_LIBRARY_PATH ${PACKAGE_PREFIX_DIR}/lib) + set_property (TARGET ${HDF5_NAMESPACE}h5dump-shared PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump-shared") + set_property (TARGET ${HDF5_NAMESPACE}h5diff-shared PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5diff-shared") + set_property (TARGET ${HDF5_NAMESPACE}h5repack-shared PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack-shared") + else () + set (HDF5_FOUND 0) + endif () + if (FORTRAN_INTERFACE AND ${HDF5_BUILD_FORTRAN}) + if (HDF5_shared_Fortran_FOUND) + set (HDF5_FORTRAN_INCLUDE_DIRS ${HDF5_INCLUDE_DIR_FORTRAN}) + set (HDF5_FORTRAN_LIBRARIES ${HDF5_FORTRAN_SHARED_LIBRARY}) + else () + set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) + message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") + endif () + else () + set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) + message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") + endif () + message (STATUS "HDF5 Tools imported location - ${HDF5_TOOLS_DIR}") + endif () + else () + if (FORTRAN_INTERFACE) + set(FORTRAN_COMP "Fortran") + endif() + find_package (HDF5 COMPONENTS ${FORTRAN_COMP}) # Legacy find + if (FORTRAN_INTERFACE AND NOT HDF5_Fortran_FOUND) + set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) + message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") + endif () + #Legacy find_package does not set HDF5_TOOLS_DIR, so we set it here + get_filename_component(HDF5_BIN_DIR ${HDF5_DIFF_EXECUTABLE} DIRECTORY) + set(HDF5_TOOLS_DIR ${HDF5_BIN_DIR}) + if (NOT TARGET hdf5::h5dump) + add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump") + set (HDF5_DUMP_EXECUTABLE $) + endif () + + if (NOT TARGET hdf5::h5diff) + add_executable (${HDF5_NAMESPACE}h5diff IMPORTED) + set_property (TARGET ${HDF5_NAMESPACE}h5diff PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5diff") + set (HDF5_DIFF_EXECUTABLE $) + endif () + + if (NOT TARGET hdf5::h5repack) + add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack") + set (HDF5_REPACK_EXECUTABLE $) + endif () + message (STATUS "HDF5 legacy Tools found - ${HDF5_DUMP_EXECUTABLE}") + endif () + + set (HDF5_PACKAGE_NAME ${SEARCH_PACKAGE_NAME}) + + if (HDF5_FOUND) + set (HDF5_HAVE_H5PUBCONF_H 1) + set (HDF5_HAVE_HDF5 1) + set (H5Z-ZFP_HDF5_HEADER "h5pubconf.h") + set (HDF5_INCLUDE_DIRS ${HDF5_INCLUDE_DIR}) + message (STATUS "HDF5-${HDF5_VERSION_STRING} found: INC=${HDF5_INCLUDE_DIR} TOOLS=${HDF5_TOOLS_DIR}") + else () + message (FATAL_ERROR " HDF5 shared is required for H5Z-ZFP") + endif () + else () + # This project is being called from within another and HDF5 is already configured + set (HDF5_HAVE_H5PUBCONF_H 1) + set (HDF5_HAVE_HDF5 1) + set (HDF5_LIBRARIES ${HDF5_LINK_LIBS}) + set (HDF5_INCLUDE_DIRS ${HDF5_INCLUDE_DIR}) + endif () + set (HDF5_LIBRARY_PATH ${PACKAGE_PREFIX_DIR}/lib) + if (FORTRAN_INTERFACE) + message (STATUS "HDF5 Fortran libs: include:${HDF5_FORTRAN_INCLUDE_DIRS} and shared:${HDF5_FORTRAN_LIBRARIES}") + endif () + message (STATUS "HDF5 link libs: ${HDF5_LIBRARIES} Includes: ${HDF5_INCLUDE_DIRS}") +endmacro () + +#------------------------------------------------------------------------------- +macro (SET_HDF_BUILD_TYPE) + if (_isMultiConfig) + # HDF_CFG_BUILD_TYPE is used in the Fortran install commands for the build location of the .mod files + set (HDF_CFG_BUILD_TYPE \${CMAKE_INSTALL_CONFIG_NAME}) + if (CMAKE_BUILD_TYPE) + # set the default to the specified command line define + set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) + else () + # set the default to the MultiConfig variable + set (HDF_CFG_NAME "$") + endif () + else () + set (HDF_CFG_BUILD_TYPE ".") + if (CMAKE_BUILD_TYPE) + set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) + else () + set (HDF_CFG_NAME "Release") + endif () + endif () + if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (VERBOSE "Setting build type to 'RelWithDebInfo' as none was specified.") + endif() + set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) + # Set the possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") + endif() +endmacro () + +#------------------------------------------------------------------------------- +macro (TARGET_C_PROPERTIES wintarget libtype) + target_compile_options(${wintarget} PRIVATE + "$<$:${WIN_COMPILE_FLAGS}>" + "$<$:${WIN_COMPILE_FLAGS}>" + ) + if(MSVC) + set_property(TARGET ${wintarget} APPEND PROPERTY LINK_FLAGS "${WIN_LINK_FLAGS}") + endif() +endmacro () + +macro (HDFTEST_COPY_FILE src dest target) + add_custom_command( + OUTPUT "${dest}" + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy_if_different "${src}" "${dest}" + DEPENDS "${src}" + ) + list (APPEND ${target}_list "${dest}") +endmacro () diff --git a/config.make b/config.make index 43112c31..13b5d962 100644 --- a/config.make +++ b/config.make @@ -1,25 +1,40 @@ export SHELL = /bin/bash -ifeq ($(HDF5_HOME),) - $(warning WARNING: HDF5_HOME not specified) -endif - -ifeq ($(ZFP_HOME),) - $(warning WARNING: ZFP_HOME not specified) -endif - -# Construct version variable depending on what dir we're in +#ifeq ($(HDF5_HOME),) +# $(warning WARNING: HDF5_HOME not specified) +#endif +# +#ifeq ($(ZFP_HOME),) +# $(warning WARNING: ZFP_HOME not specified) +#endif + +# disallow relative paths in HOME variables +HOME_WORDS := $(subst /, ,$(HDF5_HOME)) +FIRST_WORD := $(firstword $(HOME_WORDS)) +ifeq ($(FIRST_WORD),.) + $(error Please use absolute path for HDF5_HOME) +else ifeq ($(FIRST_WORD),..) + $(error Please use absolute path for HDF5_HOME) +endif +HOME_WORDS := $(subst /, ,$(ZFP_HOME)) +FIRST_WORD := $(firstword $(HOME_WORDS)) +ifeq ($(FIRST_WORD),.) + $(error Please use absolute path for ZFP_HOME) +else ifeq ($(FIRST_WORD),..) + $(error Please use absolute path for ZFP_HOME) +endif + +# Construct H5Z_ZFP_BASE variable depending on what dir this config.cmake is being included from PWD_BASE = $(shell basename $$(pwd)) -ifeq ($(PWD_BASE),src) - H5Z_ZFP_BASE := . -else ifeq ($(PWD_BASE),test) - H5Z_ZFP_BASE := ../src -else ifeq ($(PWD_BASE),H5Z-ZFP) +ifneq ($(wildcard config.make),) # we're in top-level dir H5Z_ZFP_BASE := ./src +else ifneq ($(wildcard test_write.c),) # we're in test dir + H5Z_ZFP_BASE := ../src +else ifneq ($(wildcard H5Zzfp.c),) # we're in src dir + H5Z_ZFP_BASE := . endif H5Z_ZFP_PLUGIN := $(H5Z_ZFP_BASE)/plugin -H5Z_ZFP_VERSINFO := $(shell grep '^\#define H5Z_FILTER_ZFP_VERSION_[MP]' $(H5Z_ZFP_BASE)/H5Zzfp_plugin.h | cut -d' ' -f3 | tr '\n' '.' | cut -d'.' -f-3 2>/dev/null) ZFP_HAS_REVERSIBLE := ifneq ($(ZFP_HOME),) ZFP_HAS_REVERSIBLE := $(shell grep zfp_stream_set_reversible $(ZFP_HOME)/include/zfp.h 2>/dev/null) @@ -34,7 +49,7 @@ ifeq ($(ZFP_LIB_VERSION),) ZFP_LIB_VERSION := $(shell grep '^\#define ZFP_VERSION_[MRPT]' $(ZFP_HOME)/inc/zfp.h 2>/dev/null | tr ' ' '\n' | grep '[0-9]' | tr -d '\n' 2>/dev/null) endif ifeq ($(ZFP_LIB_VERSION),) - $(warning WARNING: ZFP lib version not detected by make -- some tests may run) + $(warning WARNING: ZFP lib version not detected by make -- some tests may be skipped) endif # Detect system type @@ -51,6 +66,7 @@ HAS_ICC := $(shell basename $$(which icc 2>/dev/null) 2>/dev/null) HAS_PGCC := $(shell basename $$(which pgcc 2>/dev/null) 2>/dev/null) HAS_XLCR := $(shell basename $$(which xlc_r 2>/dev/null) 2>/dev/null) HAS_BGXLCR := $(shell basename $$(which bgxlc_r 2>/dev/null) 2>/dev/null) +HAS_CC := $(shell basename $$(which cc 2>/dev/null) 2>/dev/null) # Common Fortran compilers HAS_GFORTRAN := $(shell basename $$(which gfortran 2>/dev/null) 2>/dev/null) @@ -65,6 +81,8 @@ ifeq ($(CC),) CC = $(HAS_CLANG) else ifneq ($(strip $(HAS_GCC)),) CC = $(HAS_GCC) + else ifneq ($(strip $(HAS_CC)),) + CC = $(HAS_CC) endif else ifneq ($(findstring ppc, $(PROCESSOR),),) ifneq ($(strip $(HAS_BGXLCR)),) @@ -77,6 +95,8 @@ ifeq ($(CC),) else ifneq ($(strip $(HAS_GCC)),) CC = $(HAS_GCC) + else ifneq ($(strip $(HAS_CLANG)),) + CC = $(HAS_CLANG) else ifneq ($(strip $(HAS_ICC)),) CC = $(HAS_ICC) else ifneq ($(strip $(HAS_PGCC)),) @@ -101,7 +121,7 @@ ifneq ($(findstring gcc, $(CC)),) else ifneq ($(findstring clang, $(CC)),) SOEXT ?= dylib SHFLAG ?= -dynamiclib - PREPATH = -L + PREPATH = -Wl,-rpath, else ifneq ($(findstring icc, $(CC)),) CFLAGS += -fpic SOEXT ?= so @@ -112,6 +132,11 @@ else ifneq ($(findstring pgcc, $(CC)),) SOEXT ?= so SHFLAG ?= -shared PREPATH = -Wl,-rpath, +else ifneq ($(findstring cc, $(CC)),) + CFLAGS += -fPIC + SOEXT ?= so + SHFLAG ?= -shared + PREPATH = -Wl,-rpath, else ifneq ($(findstring xlc_r, $(CC)),) CFLAGS += -qpic SOEXT ?= so @@ -145,7 +170,7 @@ else ifneq ($(wildcard $(ZFP_HOME)/inc),) ZFP_INC = $(ZFP_HOME)/inc endif ifeq ($(wildcard $(ZFP_INC)/zfp.h),) # no header file -$(error "zfp.h not found") +$(warning "zfp.h not found") endif ifeq ($(wildcard $(ZFP_HOME)/lib),) @@ -188,6 +213,11 @@ else MAKEVARS = HDF5_HOME=$(HDF5_HOME) endif +HDF5_HAS_WRITE_CHUNK = 1 +ifeq ($(shell grep H5Dwrite_chunk $(HDF5_INC)/*.h),) + HDF5_HAS_WRITE_CHUNK = 0 +endif + ifeq ($(PREFIX),) PREFIX := $(shell pwd)/install endif diff --git a/docs/cd_vals.rst b/docs/cd_vals.rst new file mode 100644 index 00000000..d96f0aef --- /dev/null +++ b/docs/cd_vals.rst @@ -0,0 +1,40 @@ +======================================= +H5Z-ZFP and the HDF5 filter's cd_values +======================================= + +.. note:: + The details described here are likely relevant only to *developers* of the H5Z-ZFP_ filter. + If you just want to *use* the filter, you can ignore this material. + +The HDF5_ library uses an array of values, named ``cd_values`` in formal arguments documenting various API functions, for managing *auxiliary data* for a filter. +Instances of this ``cd_values`` array are used in two subtly different ways within HDF5. + +The first use is in *passing* auxiliary data for a filter from the caller to the library when initially creating a dataset. +This happens *directly* in an ``H5Pset_filter()`` (`see here `_) call. + +The second use is in *persisting* auxiliary data for a filter to the dataset's object *header* in a file. +This happens *indirectly* as part of an ``H5Dcreate()`` call. + +When a dataset creation property list includes a filter, the filter's ``set_local()`` method is called (see `H5Zregister() `_) as part of the ``H5Dcreate`` call. +In the filter's ``set_local()`` method, the ``cd_values`` that were *passed* by the caller (in ``H5Pset_filter()``) are often modified (via ``H5Pmodify_filter()`` (`see here `__) before they are *persisted* to the dataset's object header in a file. + +Among other things, this design allows a filter to be generally configured for *any* dataset in a file and then adjusted as necessary to handle such things as data type and/or dimensions when it is applied to a specific dataset. +Long story short, the data stored in ``cd_values`` of the dataset object's header in the file are often not the same values passed by the caller when the dataset was created. + +To make matters a tad more complex, the ``cd_values`` data is treated by HDF5_ as an array of C typed, 4-byte, ``unsigned integer`` values. +Furthermore, regardless of `endianness `__ of the data producer, the persisted values are always stored in little-endian format in the dataset object header in the file. +Nonetheless, if the persisted ``cd_values`` data is ever retrieved (e.g. via ``H5Pget_filter_by_id()`` (`see here `__), the HDF5_ library ensures the data is returned to callers with proper endianness. +When command-line tools like ``h5ls`` and ``h5dump`` print ``cd_values``, the data will be displayed correctly. + +Handling double precision auxiliary data via ``cd_values`` is still more complicated because a single double precision value will span multiple entries in ``cd_values`` in almost all cases. +Setting aside the possibility of differing floating point formats between the producer and consumers, any endianness handling the HDF5_ library does for the 4-byte entries in ``cd_values`` will certainly not ensure proper endianness handling of larger values. +It is impossible for command-line tools like ``h5ls`` and ``h5dump`` to display such data correctly. + +Fortunately, the ZFP_ library has already been designed to handle these issues as part of the ZFP_'s *native* stream header. +But, the ZFP_ library handles these issues in an endian-agnostic way. +Consequently, the H5Z-ZFP_ filter uses the ``cd_values`` that is persisted to a dataset's object header to store ZFP_'s stream header. +ZFP_'s stream header is stored starting at ``&cd_values[1]``. +``cd_values[0]`` is used to stored H5Z-ZFP_ filter and ZFP_ library and ZFP_ encoder version information. + +This also means that H5Z-ZFP_ avoids the overhead of duplicating the ZFP_ stream header in each dataset chunk. +For larger chunks, these savings are probably not too terribly significant. diff --git a/docs/cmake.rst b/docs/cmake.rst deleted file mode 100644 index 77c22a7f..00000000 --- a/docs/cmake.rst +++ /dev/null @@ -1,93 +0,0 @@ -===== -CMake -===== - -It is possible to build the H5Z-ZFP filter using the CMake build system. If you decide to -do so, please build ZFP also with its CMake build system. This is necessary to get the -correct dependencies from ZFP. For example, it is possible to build ZFP with OpenMP support. -The resulting CMake config files of ZFP build will make sure that this OpenMP dependency is -correctly propagated to the build of H5Z-ZFP filter. However, for HDF5 it is not -necessary to build it with its CMake build system but it is strongly recommended. - ------------------ -Compiling H5Z-ZFP ------------------ - -Similar as for the Makefile installation, the CMake build system is designed such it -compiles both the standalone HDF5_ *plugin* and a separate *library* an application -can explicitly link. See :ref:`plugin-vs-library` - -Once you have installed both HDF5 and ZFP, you can compile H5Z-ZFP_ using a command=line... - -:: - - export HDF_DIR= - export ZFP_DIR= - CC= FC= cmake -DCMAKE_INSTALL_PREFIX= - -where ```` is a directory containing ZFP_ ``inc[lude]`` and ``lib`` directories and -```` is a directory containing HDF5_ ``include`` and ``lib`` directories. Furthermore, -``src-dir`` is the directory where the source is located and ``path-to-install`` is the directory in -which the resulting *plugin* and *library* will be installed. Once cmake has finished successfully, -you can build and install the filter using the command... - -:: - - make install - -This cmake and make combination builds both the C and Fortran interface. In the case you want to specify -the ```` and ``>`` via command-line to CMake, the command looks like this... - -:: - - CC= FC= cmake -DCMAKE_INSTALL_PREFIX= - -DCMAKE_PREFIX_PATH=";" - -Please, notice the double quotes in the CMAKE_PREFIX_PATH expression. These are needed to make sure that semicolon -is interpreted as a semicolon instead of a new command. - -It is possible to build the filter without the Fortran interface. This is done as follows... - -:: - - export HDF5_DIR= - export ZFP_DIR= - CC= cmake -DCMAKE_INSTALL_PREFIX= -DFORTRAN_INTERFACE:BOOL=OFF - -followed by the same make command... - -:: - - make install - -------------------------------------------- -Including H5Z-ZFP filter in a CMake project -------------------------------------------- - -Suppose you have built the H5Z-ZFP filter using the CMake build system and installed it in ````. -To include it in another CMake project is done using the following steps. First edit the ``CMakeLists.txt`` -by adding the following two lines... - -:: - - cmake_policy(SET CMP0028 NEW) # Double colon in target name means ALIAS or IMPORTED target. - ... - set(H5Z_ZFP_USE_STATIC_LIBS OFF) - find_package(H5Z_ZFP 1.0.1 CONFIG) - ... - target_link_libraries( h5z_zfp::h5z_zfp) - ... - -where ```` in the target within the CMake project. This could be, for example, a executable or library. -Furthermore, check if the cmake version is equal or greater than 3.9. Next, you need to make sure that the filter -can be found by CMake, followed CMake itself and make... - -:: - - export H5Z_ZFP_DIR= - CC= cmake -DCMAKE_INSTALL_PREFIX= - make install - -The cmake command itself could be different depending on the CMake project you have created. If you want -to make use of the H5Z_ZFP *library* instead of the plugin, change cmake variable H5Z_ZFP_USE_STATIC_LIBS -to ON and build the project. diff --git a/docs/conf.py b/docs/conf.py index 1103b22c..2fb5ab0a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,10 +47,11 @@ copyright = u'2016, LLNL-CODE-707197' rst_epilog = """ -.. _ZFP: https://computation.llnl.gov/projects/floating-point-compression -.. _HDF5: https://support.hdfgroup.org/HDF5/doc/index.html +.. _ZFP: https://computing.llnl.gov/projects/zfp +.. _HDF5: https://docs.hdfgroup.org/hdf5/develop/ .. _H5Z-ZFP: https://github.com/LLNL/H5Z-ZFP .. _Spack: https://spack.io +.. _CMake: https://cmake.org/cmake/help/latest/ """ # The version info for the project you're documenting, acts as replacement for @@ -58,9 +59,9 @@ # built documents. # # The short X.Y version. -version = '0.6' +version = '1.1' # The full version, including alpha/beta/rc tags. -release = '0.6.0' +release = '1.1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -105,7 +106,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = "sphinx_rtd_theme" +#html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/direct.rst b/docs/direct.rst index 927b498b..1ea44e4d 100644 --- a/docs/direct.rst +++ b/docs/direct.rst @@ -22,14 +22,15 @@ code that does this, have a look at... In particular, look for the line using ``H5Dchunk_write`` in place of ``H5Dwrite``. In all other respects, the code looks the same. + The test case for this code writes uncompressed data as a dataset named ``zfparr_original``, the compressed dataset named ``zfparr_compressed`` using the filter and then the compressed -data a second time named ``zfparr_direct`` using a direct write. Then, the ``h5diff`` tool -is used to compare the data in the original and direct datasets. +data a second time named ``zfparr_direct`` using a direct_ write. Then, the ``h5diff`` tool +is used to compare the data in the original and the direct_ write datasets. Note that in order for consumers to work as normal, the producer must set dataset *creation* properties as it ordinarily would using the H5Z-ZFP_ filter. In the call to ``H5Dchunk_write``, the caller indicates to the HDF5 library not to invoke the filter via the ``filters`` mask argument. -.. _direct: https://portal.hdfgroup.org/display/HDF5/H5D_WRITE_CHUNK +.. _direct: https://docs.hdfgroup.org/hdf5/develop/group___h5_d.html#title38 diff --git a/docs/endian_issues.rst b/docs/endian_issues.rst index 34b6b0e8..b7aeeba0 100644 --- a/docs/endian_issues.rst +++ b/docs/endian_issues.rst @@ -4,28 +4,23 @@ Endian Issues ============= -The ZFP_ library writes an endian-independent stream. +This section describes some issues related to `endianness `__ of producers and consumers of the data processed by H5Z-ZFP_. +This is likely less of an issue than it once was because almost all modern CPUs are `little-endian `__. -When reading ZFP_ compressed data on a machine with a different -endian-ness than the writer, there is an unnavoidable -inefficiency. Upon reading data from disk and decompressing the read -stream with ZFP_, the correct endian-ness is returned in the result from -ZFP_ before the buffer is handed back to HDF5_ from the decompression -filter. This happens regardless of -reader and writer endian-ness incompatability. However, the HDF5_ -library is expecting to get from the decompression filter the endian-ness -of the data as it was stored to to file (typically -that of the writer machine) and expects to have to byte-swap that -buffer before returning to any endian-incompatible caller. So, in the H5Z-ZFP_ plugin, we wind up having -to un-byte-swap an already correct result read in a cross-endian context. That way, when -HDF5_ gets the data and byte-swaps it, it will produce the correct result. -There is an endian-ness test in the Makefile and two ZFP_ compressed -example datasets for big-endian and little-endian machines to test -that cross-endian reads/writes work correctly. +That being said, the ZFP_ library writes an endian-independent stream. -Finally, *endian-targetting*, that is setting the file datatype for an -endian-ness that is possibly different than the native endian-ness of -the writer (to, for example, alleviate down-stream consumers from having -to byte-swap due to endian incompatability between writer and reader) -is explicitly dis-allowed because it is not an operation that is currently -supported by the HDF5 library. +There is an unavoidable inefficiency when reading ZFP_ compressed data on a machine with a different endianness than the writer (e.g. a *mixed* endian context). Upon reading data from storage and decompressing the read stream with ZFP_, the correct endianness is returned in the result from ZFP_ before the buffer is handed back to HDF5_ from the decompression filter. +This happens regardless of reader and writer endianness incompatibility. +However, the HDF5_ library expects to get from H5Z-ZFP_ the endianness of the data as it was stored to the file on the writer machine and expects to have to byte-swap that buffer before returning to it an endian-incompatible caller. + +This means that in the H5Z-ZFP_ plugin, we wind up having to un-byte-swap an already correct result read in a cross-endian context. +That way, when HDF5_ gets the data and byte-swaps it as it is expecting to, it will produce the correct final result. +There is an endianness test in the Makefile and two ZFP_ compressed example datasets for big-endian and little-endian machines to test that cross-endian reads/writes work correctly. + +Again, because most CPUs are now little-endian and because ZFP_ became available only after the industry mostly moved away from big-endian, it is highly unlikely that this inefficiency will be triggered. + +Finally, *endian-targeting*, which is setting the file datatype for an endianness that is possibly different than the native endianness of the writer, is explicitly disallowed. +For example, data may be produced on a big-endian system, but most consumers will be little-endian. +Therefore, to alleviate downstream consumers from having to always byte-swap, it is desirable to byte-swap to little-endian when the data is written. +However, the juxtaposition of HDF5_'s type conversion and filter operations in a pipeline makes this impractical for the H5Z-ZFP_ filter. +The H5Z-ZFP_ filter will explicitly catch this condition, fail the compression and issue an error message. diff --git a/docs/h5repack.rst b/docs/h5repack.rst index 63cc205f..5171ee18 100644 --- a/docs/h5repack.rst +++ b/docs/h5repack.rst @@ -1,76 +1,69 @@ ================================== Using H5Z-ZFP Plugin with H5Repack ================================== -A convenient way to use and play with the ZFP_ filter is a *plugin* with -the HDF5_ `h5repack `_ -utility using the ``-f`` filter argument to apply ZFP to existing data in a file. +A convenient way to use and play with the ZFP_ filter is as a *plugin* with the HDF5_ `h5repack `__ utility using the ``-f`` command-line argument to apply ZFP to existing data in a file. ----------------- Patching h5repack ----------------- -Some versions of HDF5_'s ``h5repack`` utility contain a bug that prevents -proper parsing of the ``-f`` argument's option. In order to use ``h5repack`` -with ``-f`` argument as described here, you need to apply the patch from -`h5repack_parse.patch `_. -To do so, after you've downloaded and untar'd HDF5_ but before you've built -it, do something like the following using HDF5-1.8.14 as an example:: - gunzip < hdf5-1.8.14.tar.gz | tar xvf - - cd hdf5-1.8.14 - patch ./tools/h5repack/h5repack_parse.c /h5repack_parse.patch +.. warning:: + + Versions of HDF5_'s ``h5repack`` utility prior to 1.10.4 contain a bug that prevents proper parsing of the ``-f`` argument's option. + In order to use ``h5repack`` with ``-f`` argument as described here, you need to apply the patch from `h5repack_parse.patch `_. + To do so, after you've downloaded and untar'd HDF5_ but before you've built it, do something like the following using HDF5-1.8.14 as an example:: + + gunzip < hdf5-1.8.14.tar.gz | tar xvf - + cd hdf5-1.8.14 + patch ./tools/h5repack/h5repack_parse.c /h5repack_parse.patch ------------------------------------- Constructing an HDF5_ cd_values array ------------------------------------- HDF5_'s ``h5repack`` utility uses only the *generic* interface to HDF5_ filters. -Another challenge in using ``h5repack`` as described here is constructing the set -``unsigned int cd_values`` as is used in -`H5Pset_filter() `_ -required by the *generic* HDF5_ filter interface, especially because -of the type-punning (doubles as unsigned int) involved. - -**Note:** Querying an existing dataset using ``h5dump`` or ``h5ls`` to obtain -the *cd_values* stored with a ZFP_ compressed dataset -**will not provide the correct cd_values**. This is because the *cd_values* -stored in the file are different from those used in the *generic* interface -to invoke the ZFP_ filter. - -To facilitate constructing a valid ``-f`` argument to ``h5repack``, we have -created a utility program, ``print_h5repack_farg``, which is presently in the -``test`` directory and is built when tests are built. - -You can use the ``print_h5repack_farg`` utility to read a command-line -consisting of ZFP_ filter parameters you wish to use and it will output -part of the command-line needed for the ``-f`` argument to ``h5repack``. +Another challenge in using ``h5repack`` as described here is constructing the set ``unsigned int cd_values`` as is used in `H5Pset_filter() `__ required by the *generic* HDF5_ filter interface, especially because of the type-punning (doubles as unsigned int) which may be involved. + +.. note:: + + Querying an existing dataset using ``h5dump`` or ``h5ls`` to obtain the ``cd_values`` *stored* with a ZFP_ compressed dataset will not provide the correct ``cd_values`` needed to invoke the filter. + This is because the ``cd_values`` stored in the file are different from those needed in the *generic* interface to invoke the ZFP_ filter. + +To facilitate constructing a valid ``-f`` argument to ``h5repack``, there is a utility program, ``print_h5repack_farg``, which is presently in the ``test`` directory and is built when tests are built. +This program was originally written simply to facilitate testing of H5Z-ZFP filter. +It should eventually be made a first class *tool* installed with H5Z-ZFP. +However, it is presently made only as part of *testing* H5Z-ZFP. + +The ``print_h5repack_farg`` utility can be used to read a command-line consisting of ZFP_ filter parameters you wish to use and output part of the command-line needed for the ``-f`` argument to ``h5repack``. -------- Examples -------- -In the examples below, we use ``h5repack`` with the example data file, -``mesh.h5`` in the tests directory. +.. note:: + + The examples below assume H5Z-ZFP has been configured to run tests by either a preceding ``make check`` (if using a vanilla ``gmake`` build) or ``make test`` (if using a ``cmake`` build). + +In the examples below, we use ``h5repack`` with the example data file, ``mesh.h5`` in the tests directory. -To use ZFP_ filter in *rate* mode with a rate of ``4.5`` bits per value, -first, use the ``print_h5repack_farg``:: +To use ZFP_ filter in *rate* mode with a rate of ``4.5`` bits per value, first, use the ``print_h5repack_farg``:: % ./print_h5repack_farg zfpmode=1 rate=4.5 - Print cdvals for set of ZFP compression paramaters... - zfpmode=1 set zfp mode (1=rate,2=prec,3=acc,4=expert) + Print cdvals for set of ZFP compression parameters... + zfpmode=1 set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev) rate=4.5 set rate for rate mode of filter acc=0 set accuracy for accuracy mode of filter - prec=11 set precision for precision mode of zfp filter + prec=0 set precision for precision mode of zfp filter minbits=0 set minbits for expert mode of zfp filter - maxbits=4171 set maxbits for expert mode of zfp filter - maxprec=64 set maxprec for expert mode of zfp filter - minexp=-1074 set minexp for expert mode of zfp filter + maxbits=0 set maxbits for expert mode of zfp filter + maxprec=0 set maxprec for expert mode of zfp filter + minexp=0 set minexp for expert mode of zfp filter help=0 this help message h5repack -f argument... -f UD=32013,0,4,1,0,0,1074921472 -Next, cut-n-paste the ``-f UD=32013,0,4,1,0,0,1074921472`` in a command -to ``h5repack`` like so:: +Next, cut-n-paste the ``-f UD=32013,0,4,1,0,0,1074921472`` in a command to ``h5repack`` like so:: env LD_LIBRARY_PATH=:$(LD_LIBRARY_PATH) \ HDF5_PLUGIN_PATH= \ @@ -80,30 +73,27 @@ to ``h5repack`` like so:: -l VelocityX_2D:CHUNK=21x31 \ mesh.h5 mesh_repack.h5 -where the ``-l`` arguments indicate the dataset(s) to be re-packed as well -as their (new) chunking. +where the ``-l`` arguments indicate the dataset(s) to be re-packed as well as their (new) chunking. -To use ZFP_ filter in *accuracy* mode with an accuracy of ``0.075``, -first, use the ``print_h5repack_farg``:: +To use ZFP_ filter in *accuracy* mode with an accuracy of ``0.075``, first, use the ``print_h5repack_farg``:: % ./print_h5repack_farg zfpmode=3 acc=0.075 - Print cdvals for set of ZFP compression paramaters... - zfpmode=3 set zfp mode (1=rate,2=prec,3=acc,4=expert) - rate=4 set rate for rate mode of filter + Print cdvals for set of ZFP compression parameters... + zfpmode=3 set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev) + rate=3.5 set rate for rate mode of filter acc=0.075 set accuracy for accuracy mode of filter - prec=11 set precision for precision mode of zfp filter + prec=0 set precision for precision mode of zfp filter minbits=0 set minbits for expert mode of zfp filter - maxbits=4171 set maxbits for expert mode of zfp filter - maxprec=64 set maxprec for expert mode of zfp filter - minexp=-1074 set minexp for expert mode of zfp filter + maxbits=0 set maxbits for expert mode of zfp filter + maxprec=0 set maxprec for expert mode of zfp filter + minexp=0 set minexp for expert mode of zfp filter help=0 this help message h5repack -f argument... -f UD=32013,0,4,3,0,858993459,1068708659 -Next, cut-n-paste the ``-f UD=32013,0,4,3,0,858993459,1068708659`` in a command -to ``h5repack`` like so:: +Next, cut-n-paste the ``-f UD=32013,0,4,3,0,858993459,1068708659`` in a command to ``h5repack`` like so:: env LD_LIBRARY_PATH=:$(LD_LIBRARY_PATH) \ HDF5_PLUGIN_PATH= \ diff --git a/docs/hdf5_chunking.rst b/docs/hdf5_chunking.rst index e5c0259b..1120414d 100644 --- a/docs/hdf5_chunking.rst +++ b/docs/hdf5_chunking.rst @@ -4,38 +4,50 @@ HDF5_ Chunking ============== -HDF5_'s dataset `chunking`_ feature is a way to optimize data layout on disk -to support partial dataset reads by downstream consumers. This is all the more -important when compression filters are applied to datasets as it frees a consumer -from suffering the UNcompression of an entire dataset only to read a portion. - +HDF5_'s dataset `chunking`_ feature is a way to optimize data layout on disk to support partial dataset reads by downstream consumers. +This is all the more important when compression filters are applied to datasets as it frees a consumer from suffering the UNcompression of an entire dataset only to read a portion. ------------- ZFP Chunklets ------------- -When using HDF5_ `chunking`_ with ZFP_ compression, it is important to account -for the fact that ZFP_ does its work in tiny 4\ :sup:`d` chunklets of its -own where `d` is the dataset dimension (*rank* in HDF5_ parlance). This means -that that whenever possible `chunking`_ dimensions you select in HDF5_ should be -multiples of 4. When a chunk_ dimension is not a multiple of 4, ZFP_ will wind -up with partial chunklets which it will pad with useless data reducing overall -time and space efficiency of the results. - -The degree to which this may degrade performance depends on the percentage of a -chunk_ that is padded. Suppose we have 2D chunk of dimensions 27 x 101. ZFP_ will -have to treat it as 28 x 104 by padding out each dimension to the next closest -multiple of 4. The fraction of space that will wind up being wasted due to ZFP_ -chunklet padding will be (28x104-27x101) / (28x104) which is about 6.4%. On the -other hand, consider a 3D chunk that is 1024 x 1024 x 2. ZFP_ will have to treat -it as a 1024 x 1024 x 4 resulting in 50% waste. - -The latter example is potentialy very relevant when attemping to apply ZFP_ to -compress data long the *time* dimension in a large, 3D, simulation. Ordinarily, -a simulation advances one time step at a time and so needs to store in memory -only the *current* timestep. In order to give ZFP_ enough *width* in the time -dimension to satisfy the minimum chunklet dimension size of 4, the simulation -needs to keep in memory 4 timesteps. This is demonstrated in the example below. +When using HDF5_ `chunking`_ with ZFP_ compression, it is important to account for the fact that ZFP_ does its work in tiny 4\ :sup:`d` chunklets of its own where `d` is the dataset dimension (*rank* in HDF5_ parlance). +This means that whenever possible, the `chunking`_ dimensions you select in HDF5_ should be multiples of 4. +When a chunk_ dimension is not a multiple of 4, ZFP_ will wind up with partial chunklets, which will be padded with useless data, reducing the results' overall time and space efficiency. + +The degree to which this may degrade performance depends on the percentage of a chunk_ that is padded. +Suppose we have a 2D chunk of dimensions 27 x 101. +ZFP_ will have to treat it as 28 x 104 by padding out each dimension to the next closest multiple of 4. +The fraction of space that will wind up being wasted due to ZFP_ chunklet padding will be (28x104-27x101) / (28x104), which is about 6.4%. +On the other hand, consider a 3D chunk that is 1024 x 1024 x 2. +ZFP_ will have to treat it as a 1024 x 1024 x 4 resulting in 50% waste. + +The latter example is potentially very relevant when applying ZFP_ to compress data along the *time* dimension in a large, 3D, simulation. +Ordinarily, a simulation advances one time step at a time and so needs to store in memory only the *current* timestep. +However, in order to give ZFP_ enough *width* in the time dimension to satisfy the minimum chunklet dimension size of 4, the simulation +needs to keep in memory 4 timesteps. +This is demonstrated in the example below. + +-------------------- +Partial I/O Requests +-------------------- + +In any given H5Dwrite_ call, the caller has the option of writing (or reading) only a portion of the data in the dataset. +This is a *partial I/O* request. +This is handled by the ``mem_space_id`` and ``file_space_id`` arguments in an H5Dwrite_ call. + +An HDF5_ producer or consumer can issue partial I/O requests on *any* HDF5 dataset regardless of whether the dataset is compressed or not or whether the dataset has ``H5D_CONTIGUOUS`` layout. +When combining partial I/O with compression, chunk size and shape in relation to partial I/O request size and shape will have an impact on performance. + +This is particularly important in *writer* scenarios if an I/O request winds up overlapping chunks only partially. +Suppose the partially overlapped chunks exist in the file (from a previous write, for example). In that case, the HDF5_ library may wind up having to engage in *read-modify-write* operations for those chunks. + +If the partially overlapped chunks do not exist in the file, the HDF5_ library will wind up *fill-value* padding the chunks before they are written. +HDF5_'s default fill value is zero (as defined by the associated datatype). +Data producers can choose the desired fill value (see `H5Pset_fill_value `__) for a dataset, but this fill value can impact the space-performance of the compression filter. +On the other hand, if the partial chunks in one I/O request wind up getting fully filled in another, any fill value impacts on compressor performance are resolved. + +Finally, HDF5_ manages a `chunk cache `__ and `data sieving buffer `__ to help alleviate some of the I/O performance issues that can be encountered in these situations. ----------------------------- More Than 3 (or 4) Dimensions @@ -45,7 +57,7 @@ Versions of ZFP_ 0.5.3 and older support compression in only 1,2 or 3 dimensions. Versions of ZFP_ 0.5.4 and newer also support 4 dimensions. What if you have a dataset with more dimensions than ZFP_ can compress? -You can still use the H5Z-ZFP_ filter. But, in order to do so you +You can still use the H5Z-ZFP_ filter. But, in order to do so, you are *required* to chunk_ the dataset [1]_ . Furthermore, you must select a chunk_ size such that no more than 3 (or 4 for ZFP_ 0.5.4 and newer) dimensions are non-unitary (e.g. of size one). @@ -62,10 +74,10 @@ which have *extent* only in the non-unity dimensions of the chunk_. In the example below, we have a 4D array of shape ``int dims[] = {256,128,32,16};`` that we have intentionally constructed to be *smooth* in only 2 of its 4 dimensions (e.g. correlation is high in those dimensions). Because of that, we expect ZFP_ -compression to do well along those dimensions and we do no want ZFP_ to compress +compression to do well along those dimensions, and we do not want ZFP_ to compress along the other 2 dimensions. The *uncorrelated* dimensions here are dimensions with indices ``1`` (``128`` in ``dims[]``) and ``3`` (``16`` in ``dims[]``). -Thus, our chunk_ size and shape is chosoen to set the size for those dimension +Thus, our chunk_ size and shape are chosen to set the size for those dimension indices to ``1``, ``hsize_t hchunk[] = {256,1,32,1};`` .. literalinclude:: ../test/test_write.c @@ -75,40 +87,40 @@ indices to ``1``, ``hsize_t hchunk[] = {256,1,32,1};`` :end-before: End of high dimensional test What analysis process should you use to select the chunk_ shape? Depending -on what you expect in the way of access patters in downstream consumers, +on what you expect in the way of access patterns in downstream consumers, this can be a challenging question to answer. There are potentially two competing interests. One is optimizing the chunk_ size and shape for access patterns anticipated by downstream consumers. The other is optimizing the chunk_ size and shape for compression. These two interests may not be compatible and you may have to compromise between them. We illustrate the issues and -tradeoffs using an example. +trade-offs using an example. --------------------------------------------------- Compression *Along* the *State Iteration* Dimension --------------------------------------------------- -By *state iteration* dimension, we are referring to the main iteration -loop(s) of the data producer. For many PDE-based simulations, the main -iteration dimension is *time*. But, for some *outer loop* methods, the +By *state iteration* dimension, we refer to the data producer's main iteration +loop(s). For example, the main iteration dimension for many +PDE-based simulations is *time*. But, for some *outer loop* methods, the main iteration dimension(s) might be some kind of parameter study including -multiple paramaters. +multiple parameters. -The challenge here is to manage the data in a way that meets ZFP_'s +The challenge here is to manage the data to meet ZFP_'s chunklet size and shape *minimum* requirements. In any H5Dwrite_ at least 4 -*samples* along a ZFP_ compression dimension are needed or there will +*samples* along a ZFP_ compression dimension are needed, or there will be wasted space due to padding. This means that data must be *buffered* along those dimensions *before* H5Dwrite_'s can be issued. -For example, suppose you have a tensor valued field (e.g. a 3x3 matrix +For example, suppose you have a tensor-valued field (e.g. a 3x3 matrix at every *point*) over a 4D (3 spatial dimensions and 1 time dimension), regularly sampled domain? Conceptually, this is a 6 dimensional dataset -in HDF5_ with one of the dimensions (the *time* dimension) *extendible*. -You are free to define this as a 6 dimensional dataset in HDF5_. But, you +in HDF5_ with one of the dimensions (the *time* dimension) *extendable*. +So, you are free to define this as a 6 dimensional dataset in HDF5_. But, you will also have to chunk_ the dataset. You can select any chunk_ shape -you want except that no more than 3 (or 4 for ZFP_ versions 0.5.4 and +you want, except that no more than 3 (or 4 for ZFP_ versions 0.5.4 and newer) dimensions of the chunk_ can be non-unity. -In the code snipit below, we demonstrate this case. A key issue to deal +In the code snippet below, we demonstrate this case. A key issue to deal with is that because we will use ZFP_ to compress along the time dimension, this forces us to keep in memory a sufficient number of timesteps to match ZFP_'s chunklet size of 4. @@ -116,10 +128,10 @@ ZFP_'s chunklet size of 4. The code below iterates over 9 timesteps. Each of the first two groups of 4 timesteps are buffered in memory in ``tbuf``. Once 4 timesteps have been buffered, we can issue an H5Dwrite_ call doing -`hyperslab `_ -can issue an `H5Dwrite `_ -call doing `hyperslab `_ -partial I/O on the 6D, `extendible `_ +`hyperslab `__ +can issue an H5Dwrite_ +call doing `hyperslab `__ +partial I/O on the 6D, `extendable `__ dataset. But, notice that the chunk_ dimensions (line 10) are such that only 4 of the 6 dimensions are non-unity. This means ZFP_ will only ever see something to compress that is essentially 4D. @@ -127,7 +139,7 @@ compress that is essentially 4D. On the last iteration, we have only one *new* timestep. So, when we write this to ZFP_ 75% of that write will be *wasted* due to ZFP_ chunklet padding. However, if the application were to *restart* from this time and continue forward, this -*waste* will ulimately get overwritten with new timesteps. +*waste* would ultimately get overwritten with new timesteps. .. literalinclude:: ../test/test_write.c :language: c @@ -135,9 +147,9 @@ if the application were to *restart* from this time and continue forward, this :start-after: 6D Example :end-before: End of 6D Example -.. _chunking: https://support.hdfgroup.org/HDF5/doc/Advanced/Chunking/index.html -.. _chunk: https://support.hdfgroup.org/HDF5/doc/Advanced/Chunking/index.html -.. _H5Dwrite: https://support.hdfgroup.org/HDF5/doc/RM/RM_H5D.html#Dataset-Write +.. _chunking: https://portal.hdfgroup.org/display/HDF5/Chunking+in+HDF5 +.. _chunk: https://portal.hdfgroup.org/display/HDF5/Chunking+in+HDF5 +.. _H5Dwrite: https://docs.hdfgroup.org/hdf5/v1_14/group___h5_d.html#title37 .. [1] The HDF5_ library currently requires dataset chunking anyways for any dataset that has any kind of filter applied. diff --git a/docs/index.rst b/docs/index.rst index 8c83329c..48d376a3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,16 +2,10 @@ Welcome to H5Z-ZFP ================== -H5Z-ZFP_ is a compression filter for HDF5_ using the -`ZFP compression library `_, -supporting *lossy* and *lossless* compression of floating point and integer data to meet -bitrate, accuracy, and/or precision targets. The filter uses the -`registered `_ HDF5_ filter ID, ``32013``. -It supports single and double precision floating point and integer data *chunked* in 1, 2 or -3 dimensions. The filter will function on datasets of more than 3 dimensions (or 4 -dimensions for ZFP versions 0.5.4 and newer), albeit at the -possible expense of compression performance, as long as no more than 3 -(or 4) dimensions of the HDF5 dataset chunking are of size greater than 1. +H5Z-ZFP_ is a compression filter for HDF5_ using the ZFP_ compression library, supporting *lossy* and *lossless* compression of floating point and integer data to meet `bitrate `_, `accuracy `_, and/or `precision `_ targets. +The filter uses the `registered `__ HDF5_ filter ID, ``32013``. +It supports single and double precision floating point and integer data *chunked* in 1, 2 or 3 dimensions. +The filter will function on datasets of more than 3 dimensions (or 4 dimensions for ZFP_ versions 0.5.4 and newer), albeit at the possible expense of compression performance, as long as no more than 3 (or 4) dimensions of the HDF5 dataset chunking are of size greater than 1. Contents: @@ -22,8 +16,8 @@ Contents: installation interfaces hdf5_chunking + cd_vals direct h5repack endian_issues tests - cmake diff --git a/docs/installation.rst b/docs/installation.rst index bb8a4dd6..fd72dbaa 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,93 +2,65 @@ Installation ============ ------------------------------------------- -Installing via `Spack `_ ------------------------------------------- -The HDF5_ and ZFP_ libraries and the H5Z-ZFP_ plugin are all now part of the -Spack_ package manager. If you already have Spack_ installed, the easiest way to -install H5Z-ZFP_ is to simply use the Spack_ command ``spack install h5z-zfp``. -If you do not have Spack_ installed, it is very easy to install. +Three ways to install H5Z-ZFP_ are provided. +These are -:: +* With generic (GNU) :ref:`make ` +* With :ref:`CMake ` +* With :ref:`Spack ` - git clone https://github.com/llnl/spack.git - . spack/share/spack/setup-env.sh - spack install h5z-zfp +For both generic make and CMake_, you are responsible for also installing (or knowing where the installations are) the dependencies, ZFP_ and HDF5_. +For Spack_ installations, Spack_ will handle installation of dependencies as well. -By default, H5Z-ZFP_ will attempt to build with Fortran support which requires -a Fortran compiler. If you wish to exclude support for Fortran, use the command - -:: - - spack install h5z-zfp~fortran - -Note that these commands will build H5Z-ZFP_ **and** all of its dependencies including -the HDF5_ library (as well as a number of other dependencies you may not initially -expect. Be patient and let the build complete). In addition, by default, Spack_ installs -packages to directory *hashes* *within* the cloned Spack_ repository's directory tree, -``$spack/opt/spack``. You can find the resulting installed HDF5_ library with the command -``spack find -vp hdf5`` and your resulting H5Z-ZFP plugin installation with the command -``spack find -vp h5z-zfp``. If you wish to exercise more control over where Spack_ -installs things, have a look at -`configuring Spack `_ +.. _gnumake: -------------------- -Manual Installation -------------------- +--------------------------------- +Installing via Generic (GNU) Make +--------------------------------- -If Spack_ is not an option for you, information on *manually* installing is provided -here. +H5Z-ZFP_ installation supports both vanilla (`GNU `__) Make (described below) as well as :ref:`CMake `. ^^^^^^^^^^^^^ Prerequisites ^^^^^^^^^^^^^ -* `ZFP Library `_ (or from `Github `_) -* `HDF5 Library `_ +* `ZFP Library `_ (or from `Github `_) +* `HDF5 Library `_ * `H5Z-ZFP filter plugin `_ -^^^^^^^^^^^^^ -Compiling ZFP -^^^^^^^^^^^^^ - -* There is a ``Config`` file in top-level directory of the ZFP_ distribution that holds ``make`` variables - the ZFP_ Makefiles use. By default, this file is setup for a vanilla GNU compiler. If this is not the - appropriate compiler, edit ``Config`` as necessary to adjust the compiler and compilation flags. -* An important flag you **will** need to adjust in order to use the ZFP_ library with this HDF5_ filter is - the ``BIT_STREAM_WORD_TYPE`` CPP flag. To use ZFP_ with H5Z-ZFP_, the ZFP_ library **must** be compiled - with ``BIT_STREAM_WORD_TYPE`` of ``uint8``. Typically, this is achieved by including a line in ``Config`` - of the form ``DEFS += -DBIT_STREAM_WORD_TYPE=uint8``. If you attempt to use this filter with a ZFP_ - library compiled differently from this, the filter's ``can_apply`` method will always return - false. This will result in silently ignoring an HDF5_ client's request to compress data with - ZFP_. Also, be sure to see :ref:`endian-issues`. -* After you have setup ``Config``, simply run ``make`` and it will build the ZFP_ library placing - the library in a ``lib`` sub-directory and the necessary include files in ``inc[lude]`` sub-directory. +.. _zfp-config: + +^^^^^^^^^^^^^^ +Compiling ZFP_ +^^^^^^^^^^^^^^ + +* There is a ``Config`` file in top-level directory of the ZFP_ distribution that holds ``make`` variables the ZFP_ Makefiles use. By default, this file is setup for a vanilla GNU compiler. + If this is not the appropriate compiler, edit ``Config`` as necessary to adjust the compiler and compilation flags. +* An important flag you **will** need to adjust in order to use the ZFP_ library with this HDF5_ filter is the ``BIT_STREAM_WORD_TYPE`` CPP flag. + To use ZFP_ with H5Z-ZFP_, the ZFP_ library **must** be compiled with ``BIT_STREAM_WORD_TYPE`` of ``uint8``. + Typically, this is achieved by including a line in ``Config`` of the form ``DEFS += -DBIT_STREAM_WORD_TYPE=uint8``. + If you attempt to use this filter with a ZFP_ library compiled differently from this, the filter's ``can_apply`` method will always return false. + This will result in silently ignoring an HDF5_ client's request to compress data with ZFP_. + Also, be sure to see :ref:`endian-issues`. +* After you have setup ``Config``, simply run ``make`` and it will build the ZFP_ library placing the library in a ``lib`` sub-directory and the necessary include files in ``inc[lude]`` sub-directory. * For more information and details, please see the `ZFP README `_. ^^^^^^^^^^^^^^^ Compiling HDF5_ ^^^^^^^^^^^^^^^ -* If you want to be able to run the fortran tests for this filter, HDF5_ must be - configured with *both* the ``--enable-fortran`` and ``--enable-fortran2003`` - configuration switches. Otherwise, any vanilla installation of HDF5_ is acceptable. +* If you want to be able to run the fortran tests for this filter, HDF5_ must be configured with *both* the ``--enable-fortran`` and ``--enable-fortran2003`` configuration switches. + Otherwise, any vanilla installation of HDF5_ is acceptable. -* The Fortran interface to this filter *requires* a Fortran 2003 compiler - because it uses - `ISO_C_BINDING `_ - to define the Fortran interface. +* The Fortran interface to this filter *requires* a Fortran 2003 compiler because it uses `ISO_C_BINDING `_ to define the Fortran interface. -* If you are using HDF5-1.12 and wish to use the filter as a *library* (see :ref:`plugin-vs-library`), - you may need configure HDF5 with ``--disable-memory-alloc-sanity-check`` to work - around a memory management issue in HDF5. +* If you are using HDF5-1.12 and wish to use the filter as a *library* (see :ref:`plugin-vs-library`), you may need configure HDF5 with ``--disable-memory-alloc-sanity-check`` to work around a memory management issue in HDF5. ------------------ +^^^^^^^^^^^^^^^^^ Compiling H5Z-ZFP ------------------ +^^^^^^^^^^^^^^^^^ -H5Z-ZFP_ is designed to be compiled both as a standalone HDF5_ *plugin* and as a separate -*library* an application can explicitly link. See :ref:`plugin-vs-library`. +H5Z-ZFP_ is designed to be compiled both as a standalone HDF5_ *plugin* and as a separate *library* an application can explicitly link. See :ref:`plugin-vs-library`. Once you have installed the prerequisites, you can compile H5Z-ZFP_ using a command-line... @@ -98,14 +70,12 @@ Once you have installed the prerequisites, you can compile H5Z-ZFP_ using a comm ZFP_HOME= HDF5_HOME= \ PREFIX= -where ```` is a directory containing ZFP_ ``inc[lude]`` and ``lib`` dirs and -```` is a directory containing HDF5_ ``include`` and ``lib`` dirs. -If you don't specify a C compiler, it will try to guess one from your path. Fortran -compilation is optional. If you do not specify a Fortran compiler, it will not attempt -to build the Fortran interface. However, if the variable ``FC`` is already defined in -your enviornment (as in Spack_ for example), then H5Z-ZFP_ will attempt to build Fortran. -If this is not desired, the solution is to pass an *empty* ``FC`` on the make command -line as in... +where ```` is a directory containing ZFP_ ``inc[lude]`` and ``lib`` dirs and ```` is a directory containing HDF5_ ``include`` and ``lib`` dirs. +If you don't specify a C compiler, it will try to guess one from your path. +Fortran compilation is optional. +If you do not specify a Fortran compiler, it will not attempt to build the Fortran interface. +However, if the variable ``FC`` is already defined in your environment (as in Spack_ for example), then H5Z-ZFP_ will attempt to build Fortran. +If this is not desired, the solution is to pass an *empty* ``FC`` on the make command line as in... :: @@ -114,16 +84,12 @@ line as in... PREFIX= -The Makefile uses GNU Make syntax and is designed to work on OSX and -Linux. The filter has been tested on gcc, clang, xlc, icc and pgcc compilers -and checked with valgrind. +The Makefile uses GNU Make syntax and is designed to work on OSX and Linux. The filter has been tested on gcc, clang, xlc, icc and pgcc compilers and checked with valgrind. -The command ``make help`` will print useful information -about various make targets and variables. ``make check`` will compile everything -and run a handful of tests. +The command ``make help`` will print useful information about various make targets and variables. ``make check`` will compile everything and run a handful of tests. -If you don't specify a ``PREFIX``, it will install to ``./install``. The installed -package will look like... +If you don't specify a ``PREFIX``, it will install to ``./install``. +The installed package will look like... :: @@ -136,6 +102,156 @@ where ``$(PREFIX)`` resolves to whatever the full path of the installation is. To use the installed filter as an HDF5_ *plugin*, you would specify, for example, ``setenv HDF5_PLUGIN_PATH $(PREFIX)/plugin`` +.. _ceemake: + +-------------------- +Installing via CMake +-------------------- + +It is possible to build the H5Z-ZFP_ filter using the CMake_ build system. +To use CMake_ for H5Z-ZFP_, it is necessary to have also built ZFP_ with CMake. +This is necessary to get the correct dependencies from ZFP_. +For example, it is possible to build ZFP_ with OpenMP support. +The resulting CMake_ config files of ZFP_ build will make sure that this OpenMP dependency is correctly propagated to the build of H5Z-ZFP_ filter. +However, for HDF5_ it is not necessary to build it with its CMake_ build system but it is strongly recommended. + +ZFP_ must have been :ref:`configured ` with ``BIT_STREAM_WORD_TYPE`` of ``uint8`` as described above. + +Similar as for the Makefile installation, the CMake_ build system is designed such it compiles both the standalone HDF5_ *plugin* and a separate *library* an application can explicitly link. See :ref:`plugin-vs-library` + +Once both HDF5_ and ZFP_ have been installed, H5Z-ZFP_ can be compiled using a command=line... + +:: + + export HDF5_DIR= + export ZFP_DIR= + CC= FC= cmake -DCMAKE_INSTALL_PREFIX= + +where ```` is a directory containing ``zfp-config.cmake`` and ```` is a directory containing HDF5_ ``include`` and ``lib`` directories. +Furthermore, ``src-dir`` is the directory where the H5Z-ZFP_ source is located and ``path-to-install`` is the directory in which the resulting *plugin* and *library* will be installed. +Once ``cmake`` has finished successfully, you can build and install the filter using the command... + +:: + + make install + +This ``cmake`` and ``make`` combination builds both the C and Fortran interface. +In the case you want to specify the ```` and ``>`` via command-line to CMake_, the command looks like this... + +:: + + CC= FC= cmake -DCMAKE_INSTALL_PREFIX= + -DCMAKE_PREFIX_PATH=";" + +.. note:: + + The double quotes in the CMAKE_PREFIX_PATH expression are necessary to make sure that semicolon is interpreted as a semicolon instead of a new command. + +It is possible to build the filter without the Fortran interface. This is done as follows... + +:: + + export HDF5_DIR= + export ZFP_DIR= + CC= cmake -DCMAKE_INSTALL_PREFIX= -DFORTRAN_INTERFACE:BOOL=OFF + +followed by the same make command... + +:: + + make install + +------------------------------------------- +Including H5Z-ZFP filter in a CMake project +------------------------------------------- + +Suppose you have built the H5Z-ZFP_ filter using the CMake_ build system and installed it in ````. +To include it in another CMake_ project is done using the following steps. First edit the ``CMakeLists.txt`` +by adding the following two lines... + +:: + + cmake_policy(SET CMP0028 NEW) # Double colon in target name means ALIAS or IMPORTED target. + ... + set(H5Z_ZFP_USE_STATIC_LIBS OFF) + find_package(H5Z_ZFP 1.0.1 CONFIG) + ... + target_link_libraries( h5z_zfp::h5z_zfp) + ... + +where ```` in the target within the CMake_ project. +This could be, for example, an executable or library. +Furthermore, check if the ``cmake`` version is equal or greater than 3.9. +Next, you need to make sure that the filter can be found by CMake_, followed by ``cmake`` itself and ``make``... + +:: + + export H5Z_ZFP_DIR= + CC= cmake -DCMAKE_INSTALL_PREFIX= + make install + +The ``cmake`` command itself could be different depending on the CMake_ project you have created. +If you want to make use of the H5Z-ZFP_ *library* instead of the plugin, change cmake variable ``H5Z_ZFP_USE_STATIC_LIBS`` to ``ON`` and build the project. + +.. _spack2: + +--------------------- +Installing via Spack_ +--------------------- +If you already have experience with Spack_, one way to install H5Z-ZFP_ is to use the command ``spack install h5z-zfp``. +If you do not have Spack_ installed, it is easy to install. +Assuming you are working in a Bash shell...:: + + git clone https://github.com/llnl/spack.git + cd spack + git checkout releases/v0.20 + . ./share/spack/setup-env.sh + spack install h5z-zfp + +.. note:: + + It is important to work from a *released* branch of Spack_. + The command ``git checkout releases/v0.20`` ensures this. + If a newer release of Spack_ is available, by all means feel free to use it. + Just change the ``v0.20`` to indicate the release of the Spack_ you want. + The command ``git branch -r | grep releases`` will produce a list of the available release branches. + +If you are using a version of Spack_ very much older than the release of H5Z-ZFP_ you intend to use, you may have to *pin* various versions of H5Z-ZFP_, ZFP_ and/or HDF5_. +This is done by using Spack_'s ``@`` modifier to specify versions. +For example, to *pin* the version of the ZFP_ library to 0.5.5, the Spack_ command would look like:: + + spack install h5z-zfp ^zfp@0.5.5 + +To use the ``develop`` version of H5Z-ZFP_ with version 1.10.6 of HDF5_ :: + + spack install h5z-zfp@develop ^hdf5@1.10.6 + +By default, H5Z-ZFP_ will attempt to build with Fortran support which requires a Fortran compiler. +If you wish to exclude support for Fortran, use the command:: + + spack install h5z-zfp~fortran + +Spack_ packages can sometimes favor the use of dependencies you may not need. +For example, the HDF5_ package favors the use of MPI. +Since H5Z-ZFP_ depends on HDF5_, this behavior will then create a dependency of H5Z-ZFP_ on MPI. +To avoid this, you can force Spack_ to use a version of HDF5_ *without* MPI. +In the example command below, we force Spack_ to not use MPI with HDF5_ and to not use OpenMP with ZFP_:: + + spack install h5z-zfp~fortran ^hdf5~mpi~fortran ^zfp~openmp + +This can have the effect of substantially reducing the number of dependencies Spack_ winds up having to build (from 35 in one case to 10) in order to install H5Z-ZFP_ which, in turn, speeds up the install process. + +.. note:: + + Spack_ will build H5Z-ZFP_ **and** all of its dependencies including the HDF5_ library *as well as a number of other dependencies you may not initially expect*. + Be patient and let the build complete. + It may take as much as an hour. + +In addition, by default, Spack_ installs packages to directory *hashes within* the cloned Spack_ repository's directory tree, ``$spack/opt/spack``. +You can find the resulting installed HDF5_ library with the command ``spack find -vp hdf5`` and the resulting H5Z-ZFP_ plugin installation with the command ``spack find -vp h5z-zfp``. +If you wish to exercise more control over how and where Spack_ installs, have a look at +`configuring Spack `_ + -------------------------------- H5Z-ZFP Source Code Organization -------------------------------- @@ -144,21 +260,14 @@ The source code is in two separate directories * ``src`` includes the ZFP_ filter and a few header files - * ``H5Zzfp_plugin.h`` is an optional header file applications *may* wish - to include because it contains several convenient macros for easily - controlling various compression modes of the ZFP_ library (*rate*, - *precision*, *accuracy*, *expert*) via the :ref:`generic-interface`. - * ``H5Zzfp_props.h`` is a header file that contains functions to control the - filter using *temporary* :ref:`properties-interface`. Fortran callers are - *required* to use this interface. - * ``H5Zzfp_lib.h`` is a header file for applications that wish to use the filter - explicitly as a library rather than a plugin. - * ``H5Zzfp.h`` is an *all-of-the-above* header file for applications that don't - care too much about separating out the above functionalities. - - * ``test`` includes various tests. In particular ``test_write.c`` includes examples - of using both the :ref:`generic-interface` and :ref:`properties-interface`. In - addition, there is an example of how to use the filter from Fortran in ``test_rw_fortran.F90``. + * ``H5Zzfp_plugin.h`` is an optional header file applications *may* wish to include because it contains several convenient macros for easily controlling various compression modes of the ZFP_ library (*rate*, *precision*, *accuracy*, *expert*) via the :ref:`generic-interface`. + * ``H5Zzfp_props.h`` is a header file that contains functions to control the filter using *temporary* :ref:`properties-interface`. + Fortran callers are *required* to use this interface. + * ``H5Zzfp_lib.h`` is a header file for applications that wish to use the filter explicitly as a library rather than a plugin. + * ``H5Zzfp.h`` is an *all-of-the-above* header file for applications that don't care too much about separating out the above functionalities. + + * ``test`` includes various tests. In particular ``test_write.c`` includes examples of using both the :ref:`generic-interface` and :ref:`properties-interface`. + In addition, there is an example of how to use the filter from Fortran in ``test_rw_fortran.F90``. ---------------- Silo Integration @@ -177,6 +286,6 @@ In the source code for H5Z-ZFP_ this manifests as something like what is shown i In the code snippet above, note the funny ``Z`` in front of calls to various methods in the ZFP_ library. When compiling H5Z-ZFP_ normally, that ``Z`` normally resolves to the empty string. -But, when the code is compiled with ``-DAS_SILO_BUILTIN`` (which is supported and should be done *only* when ``H5Zzfp.c`` is being compiled *within* the Silo library and *next to* a version of ZFP_ that is embedded in Silo) that ``Z`` resolves to the name of a struct and struct-member dereferncing operator as in ``zfp.``. +But, when the code is compiled with ``-DAS_SILO_BUILTIN`` (which is supported and should be done *only* when ``H5Zzfp.c`` is being compiled *within* the Silo library and *next to* a version of ZFP_ that is embedded in Silo) that ``Z`` resolves to the name of a struct and struct-member dereferencing operator as in ``zfp.``. There is a similar ``B`` used for a similar purpose ahead of calls to ZFP_'s bitstream library. This is something to be aware of and to adhere to if you plan to contribute any code changes here. diff --git a/docs/interfaces.rst b/docs/interfaces.rst index 0bf2ce3a..0bd8736e 100644 --- a/docs/interfaces.rst +++ b/docs/interfaces.rst @@ -2,32 +2,18 @@ Interfaces ========== -There are two interfaces to control the filter. One uses HDF5_'s -*generic* interface via an array of ``unsigned int cd_values`` as is used -in `H5Pset_filter() `_. The other -uses HDF5_ `properties `_ -added to the `dataset creation property list `_ -used when the dataset to be compressed is being created. You can find examples of writing -HDF5_ data using both the -`generic `_ -and -`properties `__ -interfaces in -`test_write.c `_. - -The filter itself supports either interface. The filter also supports all of the -standard ZFP_ controls for affecting compression including *rate*, *precision*, -*accuracy*, *expert* and *reversible* modes. For more information and details about -these modes of controlling ZFP_ compression, please see the -`ZFP README `_. - -Finally, you should *not* attempt to combine the ZFP_ filter with any other -*byte order altering* filter such as, for example, HDF5_'s shuffle filter. -Space-performance will be ruined. This is in contrast to HDF5_'s -`deflate `_ -filter which often performs *better* when used in conjunction with the -`shuffle `_ -filter. +There are two interfaces to control the filter. +One uses HDF5_'s *generic* interface via an array of ``unsigned int cd_values`` as is used in `H5Pset_filter() `__. +The other uses HDF5_ `property lists `__ added to the `dataset creation property list `__ used when the dataset to be compressed is being created. +You can find examples of writing HDF5_ data using both the `generic `_ and `properties `__ interfaces in `test_write.c `_. + +The filter itself supports either interface. The filter also supports all of the standard ZFP_ controls for affecting compression including *rate*, *precision*, *accuracy*, *expert* and *reversible* modes. +For more information and details about these modes of controlling ZFP_ compression, please see the `ZFP README `_. + +Finally, you should *not* attempt to combine the ZFP_ filter with any other *byte order altering* filter such as, for example, HDF5_'s shuffle filter. +Space-performance will be ruined. +This is in contrast to HDF5_'s `deflate `__ filter which often performs *better* when used in conjunction with the `shuffle `__ filter. +To understand why, see the description of :ref:`endian issues `. .. _generic-interface: @@ -35,12 +21,9 @@ filter. Generic Interface ----------------- -The generic interface is the only means of controlling the H5Z-ZFP_ filter when it -is used as a -`dynamically loaded HDF5 plugin `_. +The generic interface is the only means of controlling the H5Z-ZFP_ filter when it is used as a `dynamically loaded HDF5 plugin `_. -For the generic interface, the following CPP macros are defined in -the ``H5Zzfp_plugin.h`` header file:: +For the generic interface, the following CPP macros are defined in the ``H5Zzfp_plugin.h`` header file:: H5Pset_zfp_rate_cdata(double rate, size_t cd_nelmts, unsigned int *cd_vals); H5Pset_zfp_precision_cdata(unsigned int prec, size_t cd_nelmts, unsigned int *cd_vals); @@ -50,14 +33,10 @@ the ``H5Zzfp_plugin.h`` header file:: size_t cd_nelmts, unsigned int *cd_vals); H5Pset_zfp_reversible_cdata(size_t cd_nelmts, unsigned int *cd_vals); -These macros utilize *type punning* to store the relevant ZFP_ parameters into a -sufficiently large array (>=6) of ``unsigned int cd_values``. It is up to -the caller to then call -`H5Pset_filter() `_ -with the array of cd_values constructed by one of these macros. +These macros utilize *type punning* to store the relevant ZFP_ parameters into a sufficiently large array (>=6) of ``unsigned int cd_values``. +It is up to the caller to then call `H5Pset_filter() `__ with the array of cd_values constructed by one of these macros. -Here is example code from -`test_write.c `_... +Here is example code from `test_write.c `_... .. literalinclude:: ../test/test_write.c :language: c @@ -65,11 +44,10 @@ Here is example code from :start-after: setup zfp filter via generic (cd_values) interface :end-before: #else -However, these macros are only a convenience. You do not **need** the -``H5Zzfp_plugin.h`` header file if you want to avoid using it. But, you are then -responsible for setting up the ``cd_values`` array correctly for the -filter. For reference, the ``cd_values`` array for this ZFP_ filter is -defined like so... +However, these macros are only a convenience. +You do not **need** the ``H5Zzfp_plugin.h`` header file if you want to avoid using it. +But, you are then responsible for setting up the ``cd_values`` array correctly for the filter. +For reference, the ``cd_values`` array for this ZFP_ filter is defined like so... +-----------+---------------------------------------------------------+ | | cd_values index | @@ -84,18 +62,14 @@ defined like so... +-----------+--------+--------+---------+---------+---------+---------+ | expert | 4 | unused | minbits| maxbits| maxprec| minexp | +-----------+--------+--------+---------+---------+---------+---------+ -| reversible| 5 | unused | unused | unused | unused | unsued | +| reversible| 5 | unused | unused | unused | unused | unused | +-----------+--------+--------+---------+---------+---------+---------+ A/B are high/low 32-bit words of a double. -Note that the cd_values used in the generic interface to ``H5Pset_filter()`` -are **not the same** cd_values ultimately stored to the HDF5_ dataset header -for a compressed dataset. The values are transformed in the set_local -method to use ZFP_'s internal routines for 'meta' and 'mode' data. So, -don't make the mistake of examining the values you find in a file and -think you can use those same values, for example, in an invokation of -h5repack. +Note that the cd_values used in the generic interface to ``H5Pset_filter()`` are **not the same** cd_values ultimately stored to the HDF5_ dataset header for a compressed dataset. +The values are transformed in the set_local method to use ZFP_'s internal routines for 'meta' and 'mode' data. +So, don't make the mistake of examining the values you find in a file and think you can use those same values, for example, in an invocation of h5repack. .. _properties-interface: @@ -103,8 +77,7 @@ h5repack. Properties Interface -------------------- -For the properties interface, the following functions are defined in -the ``H5Zzfp_props.h`` header file:: +For the properties interface, the following functions are defined in the ``H5Zzfp_props.h`` header file:: herr_t H5Pset_zfp_rate(hid_t dcpl_id, double rate); herr_t H5Pset_zfp_precision(hid_t dcpl_id, unsigned int prec); @@ -114,15 +87,11 @@ the ``H5Zzfp_props.h`` header file:: unsigned int maxprec, int minexp); herr_t H5Pset_zfp_reversible(hid_t dcpl_id); -These functions take a dataset creation property list, ``hid_t dcp_lid`` and -create temporary HDF5_ property -list entries to control the ZFP_ filter. Calling any of these functions -removes the effects of any previous call to any one of these functions. -In addition, calling any one of these functions also has the effect of -adding the filter to the pipeline. +These functions take a dataset creation property list, ``hid_t dcp_lid`` and create temporary HDF5_ property list entries to control the ZFP_ filter. +Calling any of these functions removes the effects of any previous call to any one of these functions. +In addition, calling any one of these functions also has the effect of adding the filter to the pipeline. -Here is example code from -`test_write.c `_... +Here is example code from `test_write.c `_... .. literalinclude:: ../test/test_write.c :language: c @@ -131,22 +100,19 @@ Here is example code from :end-before: #endif The properties interface is more type-safe than the generic interface. -However, there is no way for the implementation of the properties interface -to reside within the filter plugin itself. The properties interface requires that the caller link -with with the filter as a *library*, ``libh5zzfp.a``. The generic -interface does not require this. +However, there is no way for the implementation of the properties interface to reside within the filter plugin itself. +The properties interface requires that the caller link with with the filter as a *library*, ``libh5zzfp.a``. +The generic interface does not require this. -Note that either interface can be used whether the -filter is used as a plugin or as a library. The difference -is whether the application calls ``H5Z_zfp_initialize()`` or not. +Note that either interface can be used whether the filter is used as a plugin or as a library. +The difference is whether the application calls ``H5Z_zfp_initialize()`` or not. ----------------- Fortran Interface ----------------- -Fortran equivalents for both the properties and generic interfaces, described above, -has been added by Scot Breitenfeld of the HDF5_ group. The code that -implements the Fortran interfaces is in the file ``H5Zzfp_props_f.F90``. +Fortran equivalents for both the properties and generic interfaces, described above, has been added by Scot Breitenfeld of the HDF5_ group. +The code that implements the Fortran interfaces is in the file ``H5Zzfp_props_f.F90``. An example of its use is in ``test/test_rw_fortran.F90``. .. _plugin-vs-library: @@ -155,34 +121,21 @@ An example of its use is in ``test/test_rw_fortran.F90``. Plugin vs. Library Operation ---------------------------- -The filter is designed to be compiled for use as both a standalone HDF5_ -`dynamically loaded HDF5 plugin `_ -and as an explicitly linked *library*. -When it is used as a plugin, it is a best practice to link the ZFP_ library -into the plugin dynamic/shared object as a *static* library. Why? In so doing, -we ensure that all ZFP_ public namespace symbols remain *confined* to the plugin -so as not to interfere with any application that may be directly explicitly linking -to the ZFP_ library for other reasons. - -All HDF5_ applications are *required* -to *find* the plugin dynamic library (named ``lib*.{so,dylib}``) -in a directory specified by the enviornment -variable, ``HDF5_PLUGIN_PATH``. Currently, the HDF5 library offers -no mechanism for applications themselves to have pre-programmed -paths in which to search for a plugin. Applications are -then always vulnerable to an incorrectly specified or unspecified ``HDF5_PLUGIN_PATH`` -environment variable. - -However, the plugin can also be used explicitly as a *library*. In this case, -**do** **not** specify the ``HDF5_PLUGIN_PATH`` enviornment variable and instead -have the application link to ``libH5Zzfp.a`` in the ``lib`` dir of the installation. +The filter is designed to be compiled for use as both a standalone HDF5_ `dynamically loaded HDF5 plugin `_ and as an explicitly linked *library*. +When it is used as a plugin, it is a best practice to link the ZFP_ library into the plugin dynamic/shared object as a *static* library. +Why? In so doing, we ensure that all ZFP_ public namespace symbols remain *confined* to the plugin so as not to interfere with any application that may be directly explicitly linking to the ZFP_ library for other reasons. + +All HDF5_ applications are *required* to *find* the plugin dynamic library (named ``lib*.{so,dylib}``) in a directory specified by the environment variable, ``HDF5_PLUGIN_PATH``. +Currently, the HDF5 library offers no mechanism for applications themselves to have pre-programmed paths in which to search for a plugin. +Applications are then always vulnerable to an incorrectly specified or unspecified ``HDF5_PLUGIN_PATH`` environment variable. + +However, the plugin can also be used explicitly as a *library*. +In this case, **do** **not** specify the ``HDF5_PLUGIN_PATH`` environment variable and instead have the application link to ``libH5Zzfp.a`` in the ``lib`` dir of the installation. Instead two initialization and finalization routines are defined:: int H5Z_zfp_initialize(void); int H5Z_zfp_finalize(void); These functions are defined in the ``H5Zzfp_lib.h`` header file. -Any applications that wish to use the filter as a *library* are required to call -the initialization routine, ``H5Z_zfp_initialize()`` before the filter can be -referenced. In addition, to free up resources used by the filter, applications may -call ``H5Z_zfp_finalize()`` when they are done using the filter. +Any applications that wish to use the filter as a *library* are required to call the initialization routine, ``H5Z_zfp_initialize()`` before the filter can be referenced. +In addition, to free up resources used by the filter, applications may call ``H5Z_zfp_finalize()`` when they are done using the filter. diff --git a/docs/tests.rst b/docs/tests.rst index 0b766bb1..ddb4d406 100644 --- a/docs/tests.rst +++ b/docs/tests.rst @@ -13,7 +13,7 @@ of the filter as an explicitly linked library. By default, these test a simple 1D array with and without ZFP_ compression using either the :ref:`generic-interface` (for plugin) or the :ref:`properties-interface` (for library). You can use the code there as an example of using the ZFP_ filter either as a plugin or as a library. However, these -also include some advanced usages for 4D and 6D, time-varying (e.g. *extendible*) +also include some advanced usages for 4D and 6D, time-varying (e.g. *extendable*) datasets. The command ``test_write_lib help`` or ``test_write_plugin help`` will print a list of the example's options and how to use them. @@ -34,7 +34,7 @@ Write Test Options chunk=256 set chunk size for 1D dataset doint=0 also do integer 1D data - ZFP compression paramaters... + ZFP compression parameters... zfpmode=3 (1=rate,2=prec,3=acc,4=expert,5=reversible) rate=4 set rate for rate mode of filter acc=0 set accuracy for accuracy mode of filter @@ -56,7 +56,7 @@ on a 4D dataset where two of the 4 dimensions are not correlated. This tests the plugin's ability to properly set chunking for HDF5 such that chunks span **only** correlated dimensions and have non-unity sizes in 3 or fewer dimensions. The ``sixd`` -test runs a test on a 6D, extendible dataset representing an +test runs a test on a 6D, extendable dataset representing an example of using ZFP_ for compression along the *time* axis. There is a companion, `test_read.c `_ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ded6e4e4..efeab5fc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,15 +19,23 @@ add_library(h5z_zfp_static STATIC ${h5z_zfp_lib_source}) target_compile_definitions(h5z_zfp_static PRIVATE H5Z_ZFP_AS_LIB) target_include_directories(h5z_zfp_static PUBLIC - $ - $ - ${HDF5_INCLUDE_DIRS}) -target_link_libraries(h5z_zfp_static PUBLIC zfp::zfp ${HDF5_LIBRARIES}) + "$" + "$" + "$" + ${HDF5_INCLUDE_DIRS} + "$<$:${HDF5_FORTRAN_INCLUDE_DIRS}>" +) +target_link_libraries(h5z_zfp_static PUBLIC zfp::zfp ${HDF5_LIBRARIES} $<$:${HDF5_FORTRAN_LIBRARIES}>) set_target_properties(h5z_zfp_static PROPERTIES POSITION_INDEPENDENT_CODE 1) -set_target_properties(h5z_zfp_static PROPERTIES OUTPUT_NAME h5zzfp) +set_target_properties(h5z_zfp_static PROPERTIES + OUTPUT_NAME h5zzfp + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) install(TARGETS h5z_zfp_static - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib) + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) #------------------------------------------------------------------------------# # Shared library @@ -41,11 +49,17 @@ add_library(h5z_zfp_shared SHARED ${h5z_zfp_plugin_source}) target_include_directories(h5z_zfp_shared PUBLIC $ + $ $ ${HDF5_INCLUDE_DIRS}) target_link_libraries(h5z_zfp_shared PUBLIC zfp::zfp ${HDF5_LIBRARIES}) -set_target_properties(h5z_zfp_shared PROPERTIES OUTPUT_NAME h5zzfp) +set_target_properties(h5z_zfp_shared PROPERTIES + OUTPUT_NAME h5zzfp + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/plugin + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/plugin + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/plugin) install(TARGETS h5z_zfp_shared + RUNTIME DESTINATION plugin ARCHIVE DESTINATION plugin LIBRARY DESTINATION plugin) @@ -56,6 +70,6 @@ install(FILES ${h5z_zfp_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) if (FORTRAN_INTERFACE) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/h5zzfp_props_f.mod + install(FILES ${CMAKE_BINARY_DIR}/bin/h5zzfp_props_f.mod DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) endif () diff --git a/src/H5Zzfp.c b/src/H5Zzfp.c index c9ea896e..523f1fbf 100644 --- a/src/H5Zzfp.c +++ b/src/H5Zzfp.c @@ -461,11 +461,6 @@ get_zfp_info_from_cd_values(size_t cd_nelmts, unsigned int const *cd_values, /* Do a read of *just* magic to detect possible codec version mismatch */ if (0 == (Z zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC))) - H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "ZFP codec version mismatch"); - Z zfp_stream_rewind(zstr); - - /* Now, read ZFP *full* header */ - if (0 == (Z zfp_read_header(zstr, zfld, ZFP_HEADER_FULL))) { herr_t conv; @@ -479,9 +474,14 @@ get_zfp_info_from_cd_values(size_t cd_nelmts, unsigned int const *cd_values, H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "header endian-swap failed"); Z zfp_stream_rewind(zstr); - if (0 == (Z zfp_read_header(zstr, zfld, ZFP_HEADER_FULL))) - H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "reading header failed"); + if (0 == (Z zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC))) + H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "ZFP codec version mismatch"); } + Z zfp_stream_rewind(zstr); + + /* Now, read ZFP *full* header */ + if (0 == (Z zfp_read_header(zstr, zfld, ZFP_HEADER_FULL))) + H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "reading header failed"); /* Get ZFP stream mode and field meta */ *zfp_mode = Z zfp_stream_mode(zstr); diff --git a/src/H5Zzfp_plugin.h b/src/H5Zzfp_plugin.h index a8a906e1..dd893c08 100644 --- a/src/H5Zzfp_plugin.h +++ b/src/H5Zzfp_plugin.h @@ -25,15 +25,14 @@ CD[0]=CD[1]=CD[2]=CD[3]=0; \ CD[0]=H5Z_ZFP_MODE_RATE; *p=R; N=4;}} while(0) #define H5Pget_zfp_rate_cdata(N, CD) \ -((double)(((N>=4)&&(CD[0]==H5Z_ZFP_MODE_RATE))?*((double *) &CD[2]):-1)) +((double)(((N>=4)&&(CD[0]==H5Z_ZFP_MODE_RATE))?*((double *) &CD[2]):0)) #define H5Pset_zfp_precision_cdata(P, N, CD) \ -do { if (N>=3) {CD[0]=CD[1]=CD[2]; \ -CD[0]=H5Z_ZFP_MODE_PRECISION; \ -CD[2]=P; N=3;}} while(0) +do { if (N>=3) {CD[0]=H5Z_ZFP_MODE_PRECISION; \ +CD[1]=0; CD[2]=P; N=3;}} while(0) #define H5Pget_zfp_precision_cdata(N, CD) \ -((double)(((N>=3)&&(CD[0]==H5Z_ZFP_MODE_ACCURACY))?CD[2]:-1)) +((double)(((N>=3)&&(CD[0]==H5Z_ZFP_MODE_PRECISION))?CD[2]:0)) #define H5Pset_zfp_accuracy_cdata(A, N, CD) \ do { if (N>=4) {double *p = (double *) &CD[2]; \ @@ -41,7 +40,7 @@ CD[0]=CD[1]=CD[2]=CD[3]=0; \ CD[0]=H5Z_ZFP_MODE_ACCURACY; *p=A; N=4;}} while(0) #define H5Pget_zfp_accuracy_cdata(N, CD) \ -((double)(((N>=4)&&(CD[0]==H5Z_ZFP_MODE_ACCURACY))?*((double *) &CD[2]):-1)) +((double)(((N>=4)&&(CD[0]==H5Z_ZFP_MODE_ACCURACY))?*((double *) &CD[2]):0)) #define H5Pset_zfp_expert_cdata(MiB, MaB, MaP, MiE, N, CD) \ do { if (N>=6) { CD[0]=CD[1]=CD[2]=CD[3]=CD[4]=CD[5]=0; \ @@ -66,6 +65,6 @@ do { if (N>=1) { \ CD[0]=H5Z_ZFP_MODE_REVERSIBLE; N=1;}} while(0) #define H5Pget_zfp_reversible_cdata(N, CD) \ -((int)(((N>=1)&&(CD[0]==H5Z_ZFP_MODE_REVERSIBLE))?1:-1)) +((int)(((N>=1)&&(CD[0]==H5Z_ZFP_MODE_REVERSIBLE))?1:0)) #endif diff --git a/src/H5Zzfp_props.c b/src/H5Zzfp_props.c index bcf16a91..8cb9b288 100644 --- a/src/H5Zzfp_props.c +++ b/src/H5Zzfp_props.c @@ -5,6 +5,7 @@ #include #include +#include #define H5Z_ZFP_PUSH_AND_GOTO(MAJ, MIN, RET, MSG) \ do \ @@ -18,14 +19,14 @@ do \ static herr_t H5Pset_zfp(hid_t plist, int mode, ...) { static char const *_funcname_ = "H5Pset_zfp"; - static size_t ctrls_sz = sizeof(h5z_zfp_controls_t); + static size_t const ctrls_sz = sizeof(h5z_zfp_controls_t); unsigned int flags; size_t cd_nelmts = 0; unsigned int cd_values[1]; h5z_zfp_controls_t *ctrls_p = 0; int i; va_list ap; - herr_t retval; + herr_t retval = 0; if (0 >= H5Pisa_class(plist, H5P_DATASET_CREATE)) H5Z_ZFP_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "not a dataset creation property list class"); @@ -101,6 +102,9 @@ static herr_t H5Pset_zfp(hid_t plist, int mode, ...) retval = H5Pset(plist, "zfp_controls", ctrls_p); } + /* HDF5 copies the memory we gave it */ + free(ctrls_p); + return retval; done: diff --git a/src/H5Zzfp_props_f.F90 b/src/H5Zzfp_props_f.F90 index 5c48b16f..e9bfa937 100644 --- a/src/H5Zzfp_props_f.F90 +++ b/src/H5Zzfp_props_f.F90 @@ -1,4 +1,4 @@ -MODULE H5Zzfp_props_f +MODULE h5zzfp_props_f #include "H5Zzfp_version.h" @@ -13,7 +13,7 @@ MODULE H5Zzfp_props_f INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MAJOR_F=H5Z_FILTER_ZFP_VERSION_MAJOR INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MINOR_F=H5Z_FILTER_ZFP_VERSION_MINOR INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_PATCH_F=H5Z_FILTER_ZFP_VERSION_PATCH - + INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MEM_F=H5Z_ZFP_CD_NELMTS_MEM INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MAX_F=H5Z_ZFP_CD_NELMTS_MAX @@ -43,7 +43,7 @@ MODULE H5Zzfp_props_f INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MAJOR=H5Z_FILTER_ZFP_VERSION_MAJOR_F INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MINOR=H5Z_FILTER_ZFP_VERSION_MINOR_F INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_PATCH=H5Z_FILTER_ZFP_VERSION_PATCH_F - + INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MEM=H5Z_ZFP_CD_NELMTS_MEM_F INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MAX=H5Z_ZFP_CD_NELMTS_MAX_F @@ -77,7 +77,7 @@ INTEGER(C_INT) FUNCTION H5Pset_zfp_precision(plist, prec) BIND(C, NAME='H5Pset_z INTEGER(HID_T), VALUE :: plist INTEGER(C_INT), VALUE :: prec END FUNCTION H5Pset_zfp_precision - + INTEGER(C_INT) FUNCTION H5Pset_zfp_accuracy(plist, acc) BIND(C, NAME='H5Pset_zfp_accuracy') IMPORT :: C_INT, C_DOUBLE, HID_T IMPLICIT NONE diff --git a/src/H5Zzfp_version.h b/src/H5Zzfp_version.h index 9bd327dc..a79179ca 100644 --- a/src/H5Zzfp_version.h +++ b/src/H5Zzfp_version.h @@ -5,7 +5,7 @@ #define H5Z_FILTER_ZFP_VERSION_MAJOR 1 #define H5Z_FILTER_ZFP_VERSION_MINOR 1 -#define H5Z_FILTER_ZFP_VERSION_PATCH 0 +#define H5Z_FILTER_ZFP_VERSION_PATCH 1 #define H5Z_ZFP_MODE_RATE 1 #define H5Z_ZFP_MODE_PRECISION 2 diff --git a/src/Makefile b/src/Makefile index 2e156e84..35cb6d21 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,4 +1,6 @@ -include ../config.make +ifneq ($(strip $(MAKECMDGOALS)),clean) + include ../config.make +endif .PHONY: all lib clean diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 00000000..e15d30b0 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,958 @@ +set(ZFP_LIB_VERSION ${ZFP_VERSION_MAJOR}${ZFP_VERSION_MINOR}${ZFP_VERSION_PATCH}) +if (ZFP_VERSION VERSION_GREATER_EQUAL 1.0.0) + set(ZFP_LIB_VERSION ${ZFP_LIB_VERSION}${ZFP_VERSION_TWEAK}) +endif () +set(ZFP_HAS_CFP 0) +if (ZFP_CFP_ENABLED) + set(ZFP_HAS_CFP 1) +endif() + +# HDF5 docs say H5Dwrite_chunk was introduced in 1.10.2 +set(HDF5_HAS_WRITE_CHUNK 0) +if ("${HDF5_VERSION}" VERSION_GREATER_EQUAL "1.10.2") + set(HDF5_HAS_WRITE_CHUNK 1) +endif() + +#------------------------------------------------------------------------------# +# Setup HDF5_PLUGIN_PATH +#------------------------------------------------------------------------------# +get_target_property(HDF5_PLUGIN_PATH h5z_zfp_shared RUNTIME_OUTPUT_DIRECTORY) +if (_isMultiConfig) + set(HDF5_PLUGIN_PATH "${HDF5_PLUGIN_PATH}/${HDF_CFG_NAME}") +endif () + +#------------------------------------------------------------------------------# +# Plugin and library write & read tests (No check implemented) +#------------------------------------------------------------------------------# +add_executable(test_write_plugin test_write.c) +target_compile_definitions(test_write_plugin PRIVATE H5Z_ZFP_USE_PLUGIN) +target_compile_definitions(test_write_plugin PRIVATE ZFP_LIB_VERSION=0x${ZFP_LIB_VERSION}) +target_link_libraries(test_write_plugin h5z_zfp_shared) + +add_executable(test_read_plugin test_read.c) +target_compile_definitions(test_read_plugin PRIVATE H5Z_ZFP_USE_PLUGIN) +target_link_libraries(test_read_plugin h5z_zfp_shared) + +add_executable(test_write_lib test_write.c) +target_compile_definitions(test_write_lib PRIVATE ZFP_LIB_VERSION=0x${ZFP_LIB_VERSION}) +target_compile_definitions(test_write_lib PRIVATE HDF5_HAS_WRITE_CHUNK=${HDF5_HAS_WRITE_CHUNK}) +target_compile_definitions(test_write_lib PRIVATE ZFP_HAS_CFP=${ZFP_HAS_CFP}) +target_link_libraries(test_write_lib h5z_zfp_static) +if (ZFP_CFP_ENABLED) + target_link_libraries(test_write_lib zfp::cfp) +endif () + +add_executable(test_read_lib test_read.c) +target_link_libraries(test_read_lib h5z_zfp_static) + +if (FORTRAN_INTERFACE) + add_executable(test_rw_fortran test_rw_fortran.F90) + target_link_libraries(test_rw_fortran h5z_zfp_static) + set_target_properties (test_rw_fortran PROPERTIES LINKER_LANGUAGE Fortran) +endif () + +#------------------------------------------------------------------------------# +# Tests +#------------------------------------------------------------------------------# + +if (MSVC) + string(REPLACE ";" "\\;" ESCAPED_PATH "$ENV{PATH}") +endif () + +# Rate +set(RATE_RATE 32 16 8 4) +if (MSVC) + set(RATE_DIFF 3e-7 0.008 2.0 45) +else () + set(RATE_DIFF 1e-7 0.003 0.4 33) +endif () +list(LENGTH RATE_RATE NRATES) +math(EXPR NRATES "${NRATES} - 1") +foreach (IRATE RANGE ${NRATES}) + list(GET RATE_RATE ${IRATE} RATE) + list(GET RATE_DIFF ${IRATE} DIFF) + + add_test( + NAME + test-rate-write-${RATE} + COMMAND + $ zfpmode=1 rate=${RATE} ofile=test_zfp-rate-${RATE}.h5 + ) + set_property( + TEST + test-rate-write-${RATE} + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-rate-write-${RATE} + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + + add_test( + NAME + test-rate-read-${RATE} + COMMAND + $ max_absdiff=${DIFF} max_reldiff=${DIFF} ifile=test_zfp-rate-${RATE}.h5 + ) + set_property( + TEST + test-rate-read-${RATE} + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-rate-read-${RATE} + APPEND + PROPERTY + ENVIRONMENT "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-rate-read-${RATE} PROPERTIES DEPENDS test-rate-write-${RATE}) + + add_test( + NAME + test-rate-cleanup-${RATE} + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-rate-${RATE}.h5 + ) + set_tests_properties (test-rate-cleanup-${RATE} PROPERTIES DEPENDS test-rate-read-${RATE}) + + set_tests_properties(test-rate-write-${RATE} PROPERTIES FIXTURES_SETUP "rate-read-${RATE};rate-cleanup-${RATE}") + set_tests_properties(test-rate-write-${RATE} PROPERTIES FIXTURES_REQUIRED "rate-write-${RATE}") + set_tests_properties(test-rate-read-${RATE} PROPERTIES FIXTURES_REQUIRED "rate-read-${RATE}") + set_tests_properties(test-rate-cleanup-${RATE} PROPERTIES FIXTURES_REQUIRED "rate-cleanup-${RATE}") + set_tests_properties(test-rate-cleanup-${RATE} PROPERTIES FIXTURES_CLEANUP "rate-write-${RATE};rate-read-${RATE}") + + add_test( + NAME + test-lib-rate-write-${RATE} + COMMAND + $ zfpmode=1 rate=${RATE} ofile=test_zfp-lib-rate-${RATE}.h5 + ) + + add_test( + NAME + test-lib-rate-read-${RATE} + COMMAND + $ max_absdiff=${DIFF} max_reldiff=${DIFF} ifile=test_zfp-lib-rate-${RATE}.h5 + ) + set_tests_properties (test-lib-rate-read-${RATE} PROPERTIES DEPENDS test-lib-rate-write-${RATE}) + + add_test( + NAME + test-lib-rate-cleanup-${RATE} + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-lib-rate-${RATE}.h5 + ) + set_tests_properties (test-lib-rate-cleanup-${RATE} PROPERTIES DEPENDS test-lib-rate-read-${RATE}) + + set_tests_properties(test-lib-rate-write-${RATE} PROPERTIES FIXTURES_SETUP "lib-rate-read-${RATE};lib-rate-cleanup-${RATE}") + set_tests_properties(test-lib-rate-write-${RATE} PROPERTIES FIXTURES_REQUIRED "lib-rate-write-${RATE}") + set_tests_properties(test-lib-rate-read-${RATE} PROPERTIES FIXTURES_REQUIRED "lib-rate-read-${RATE}") + set_tests_properties(test-lib-rate-cleanup-${RATE} PROPERTIES FIXTURES_REQUIRED "lib-rate-cleanup-${RATE}") + set_tests_properties(test-lib-rate-cleanup-${RATE} PROPERTIES FIXTURES_CLEANUP "lib-rate-write-${RATE};lib-rate-read-${RATE}") + + if (NOT MSVC AND FORTRAN_INTERFACE) + add_test( + NAME + test-rate-write-${RATE}-f + COMMAND + $ zfpmode 1 rate ${RATE} ofile test_zfp_fortran-rate-${RATE}.h5 + ) + + add_test( + NAME + test-rate-info1-${RATE}-f + COMMAND + "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${HDF5_DUMP_EXECUTABLE}" + -D "TEST_FILE=test_zfp_fortran-rate-${RATE}.h5" + -D "TEST_DSET=compressed" + -D "TEST_RATE=${RATE}" + -D "RATE_START=64" + -P "${CMAKE_CURRENT_SOURCE_DIR}/h5dump-rate.cmake" + ) + set_tests_properties (test-rate-info1-${RATE}-f PROPERTIES DEPENDS test-rate-write-${RATE}-f) + + add_test( + NAME + test-rate-info2-${RATE}-f + COMMAND + "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${HDF5_DUMP_EXECUTABLE}" + -D "TEST_FILE=test_zfp_fortran-rate-${RATE}.h5" + -D "TEST_DSET=compressed-plugin" + -D "TEST_RATE=${RATE}" + -D "RATE_START=64" + -P "${CMAKE_CURRENT_SOURCE_DIR}/h5dump-rate.cmake" + ) + set_tests_properties (test-rate-info2-${RATE}-f PROPERTIES DEPENDS test-rate-info1-${RATE}-f) + + add_test( + NAME + test-rate-cleanup-${RATE}-f + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp_fortran-rate-${RATE}.h5 + ) + set_tests_properties (test-rate-cleanup-${RATE}-f PROPERTIES DEPENDS test-rate-info2-${RATE}-f) + + set_tests_properties(test-rate-write-${RATE}-f PROPERTIES FIXTURES_SETUP "rate-info1-${RATE}-f;rate-info1-${RATE}-f;rate-cleanup-${RATE}-f") + set_tests_properties(test-rate-write-${RATE}-f PROPERTIES FIXTURES_REQUIRED "rate-write-${RATE}-f") + set_tests_properties(test-rate-info1-${RATE}-f PROPERTIES FIXTURES_REQUIRED "rate-info1-${RATE}-f") + set_tests_properties(test-rate-info2-${RATE}-f PROPERTIES FIXTURES_REQUIRED "rate-info2-${RATE}-f") + set_tests_properties(test-rate-cleanup-${RATE}-f PROPERTIES FIXTURES_REQUIRED "rate-cleanup-${RATE}-f") + set_tests_properties(test-rate-cleanup-${RATE}-f PROPERTIES FIXTURES_CLEANUP "rate-write-${RATE}-f;rate-info1-${RATE}-f;rate-info1-${RATE}-f") + endif () +endforeach () + +# Accuracy +set(ACC_NO 1 2 3 4) +set(ACC_ACC 0.1 0.01 0.001 0.0001) +set(ACC_DIFF 0.025 0.004 0.0006 4e-5) +list(LENGTH ACC_NO NACCS) +math(EXPR NACCS "${NACCS} - 1") +foreach (IACC RANGE ${NACCS}) + list(GET ACC_NO ${IACC} NUMBER) + list(GET ACC_ACC ${IACC} ACC) + list(GET ACC_DIFF ${IACC} DIFF) + + add_test( + NAME + test-accuracy-write-${NUMBER} + COMMAND + $ zfpmode=3 acc=${ACC} ofile=test_zfp-acc-${NUMBER}.h5 + ) + set_property( + TEST + test-accuracy-write-${NUMBER} + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-accuracy-write-${NUMBER} + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + + add_test( + NAME + test-accuracy-read-${NUMBER} + COMMAND + $ ret=1 max_absdiff=${DIFF} ifile=test_zfp-acc-${NUMBER}.h5 + ) + set_property( + TEST + test-accuracy-read-${NUMBER} + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-accuracy-read-${NUMBER} + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-accuracy-read-${NUMBER} PROPERTIES DEPENDS test-accuracy-write-${NUMBER}) + + add_test( + NAME + test-accuracy-cleanup-${NUMBER} + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-acc-${NUMBER}.h5 + ) + set_tests_properties (test-accuracy-cleanup-${NUMBER} PROPERTIES DEPENDS test-accuracy-read-${NUMBER}) + + set_tests_properties(test-accuracy-write-${NUMBER} PROPERTIES FIXTURES_SETUP "accuracy-read-${NUMBER};accuracy-cleanup-${NUMBER}") + set_tests_properties(test-accuracy-write-${NUMBER} PROPERTIES FIXTURES_REQUIRED "accuracy-write-${NUMBER}") + set_tests_properties(test-accuracy-read-${NUMBER} PROPERTIES FIXTURES_REQUIRED "accuracy-read-${NUMBER}") + set_tests_properties(test-accuracy-cleanup-${NUMBER} PROPERTIES FIXTURES_REQUIRED "accuracy-cleanup-${NUMBER}") + set_tests_properties(test-accuracy-cleanup-${NUMBER} PROPERTIES FIXTURES_CLEANUP "accuracy-write-${NUMBER};accuracy-read-${NUMBER}") + + add_test( + NAME + test-lib-accuracy-write-${NUMBER} + COMMAND + $ zfpmode=3 acc=${ACC} ofile=test_zfp-lib-acc-${NUMBER}.h5 + ) + + add_test( + NAME + test-lib-accuracy-read-${NUMBER} + COMMAND + $ ret=1 max_absdiff=${DIFF} ifile=test_zfp-lib-acc-${NUMBER}.h5 + ) + set_tests_properties (test-lib-accuracy-read-${NUMBER} PROPERTIES DEPENDS test-lib-accuracy-write-${NUMBER}) + + add_test( + NAME + test-lib-accuracy-cleanup-${NUMBER} + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-lib-acc-${NUMBER}.h5 + ) + set_tests_properties (test-lib-accuracy-cleanup-${NUMBER} PROPERTIES DEPENDS test-lib-accuracy-read-${NUMBER}) + + set_tests_properties(test-lib-accuracy-write-${NUMBER} PROPERTIES FIXTURES_SETUP "lib-accuracy-read-${NUMBER};lib-accuracy-cleanup-${NUMBER}") + set_tests_properties(test-lib-accuracy-write-${NUMBER} PROPERTIES FIXTURES_REQUIRED "lib-accuracy-write-${NUMBER}") + set_tests_properties(test-lib-accuracy-read-${NUMBER} PROPERTIES FIXTURES_REQUIRED "lib-accuracy-read-${NUMBER}") + set_tests_properties(test-lib-accuracy-cleanup-${NUMBER} PROPERTIES FIXTURES_REQUIRED "lib-accuracy-cleanup-${NUMBER}") + set_tests_properties(test-lib-accuracy-cleanup-${NUMBER} PROPERTIES FIXTURES_CLEANUP "lib-accuracy-write-${NUMBER};lib-accuracy-read-${NUMBER}") + + if (FORTRAN_INTERFACE) + add_test( + NAME + test-accuracy-write-${NUMBER}-f + COMMAND + $ zfpmode 3 acc ${ACC} ofile test_zfp_fortran-acc-${NUMBER}.h5 write + ) + + add_test( + NAME + test-accuracy-diff1-${NUMBER}-f + COMMAND + ${HDF5_DIFF_EXECUTABLE} -v -d ${ACC} test_zfp_fortran-acc-${NUMBER}.h5 test_zfp_fortran-acc-${NUMBER}.h5 compressed original + ) + set_property( + TEST + test-accuracy-diff1-${NUMBER}-f + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-accuracy-diff1-${NUMBER}-f + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-accuracy-diff1-${NUMBER}-f PROPERTIES DEPENDS test-accuracy-write-${NUMBER}-f) + + add_test( + NAME + test-accuracy-diff2-${NUMBER}-f + COMMAND + ${HDF5_DIFF_EXECUTABLE} -v -d ${ACC} test_zfp_fortran-acc-${NUMBER}.h5 test_zfp_fortran-acc-${NUMBER}.h5 compressed-plugin original + ) + set_property( + TEST + test-accuracy-diff2-${NUMBER}-f + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-accuracy-diff2-${NUMBER}-f + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-accuracy-diff2-${NUMBER}-f PROPERTIES DEPENDS test-accuracy-diff1-${NUMBER}-f) + + add_test( + NAME + test-accuracy-cleanup-${NUMBER}-f + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp_fortran-acc-${NUMBER}.h5 + ) + set_tests_properties (test-accuracy-cleanup-${NUMBER}-f PROPERTIES DEPENDS test-accuracy-diff2-${NUMBER}-f) + + set_tests_properties(test-accuracy-write-${NUMBER}-f PROPERTIES FIXTURES_SETUP "accuracy-diff1-${NUMBER}-f;accuracy-diff2-${NUMBER}-f;accuracy-cleanup-${NUMBER}-f") + set_tests_properties(test-accuracy-write-${NUMBER}-f PROPERTIES FIXTURES_REQUIRED "accuracy-write-${NUMBER}-f") + set_tests_properties(test-accuracy-diff1-${NUMBER}-f PROPERTIES FIXTURES_REQUIRED "accuracy-diff1-${NUMBER}-f") + set_tests_properties(test-accuracy-diff2-${NUMBER}-f PROPERTIES FIXTURES_REQUIRED "accuracy-diff2-${NUMBER}-f") + set_tests_properties(test-accuracy-cleanup-${NUMBER}-f PROPERTIES FIXTURES_REQUIRED "accuracy-cleanup-${NUMBER}-f") + set_tests_properties(test-accuracy-cleanup-${NUMBER}-f PROPERTIES FIXTURES_CLEANUP "accuracy-write-${NUMBER}-f;accuracy-diff1-${NUMBER}-f;accuracy-diff2-${NUMBER}-f") + endif () +endforeach () + +# Precision +set(PREC_PREC 12 16 20 24) + +if (MSVC) + set(PREC_DIFF 0.06 0.003 7e-4 5e-5) +else () + set(PREC_DIFF 0.02 0.0005 5e-5 1e-6) +endif () +list(LENGTH PREC_PREC NPRECS) +math(EXPR NPRECS "${NPRECS} - 1") +foreach (IPREC RANGE ${NPRECS}) + list(GET PREC_PREC ${IPREC} PREC) + list(GET PREC_DIFF ${IPREC} DIFF) + + add_test( + NAME + test-precision-write-${PREC} + COMMAND + $ zfpmode=2 prec=${PREC} ofile=test_zfp-prec-${PREC}.h5 + ) + set_property( + TEST + test-precision-write-${PREC} + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-precision-write-${PREC} + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + + add_test( + NAME + test-precision-read-${PREC} + COMMAND + $ ret=2 max_reldiff=${DIFF} ifile=test_zfp-prec-${PREC}.h5 + ) + set_property( + TEST + test-precision-read-${PREC} + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-precision-read-${PREC} + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-precision-read-${PREC} PROPERTIES DEPENDS test-precision-write-${PREC}) + + add_test( + NAME + test-precision-cleanup-${PREC} + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-prec-${PREC}.h5 + ) + set_tests_properties (test-precision-cleanup-${PREC} PROPERTIES DEPENDS test-precision-read-${PREC}) + + set_tests_properties(test-precision-write-${PREC} PROPERTIES FIXTURES_SETUP "precision-read-${PREC};precision-cleanup-${PREC}") + set_tests_properties(test-precision-write-${PREC} PROPERTIES FIXTURES_REQUIRED "precision-write-${PREC}") + set_tests_properties(test-precision-read-${PREC} PROPERTIES FIXTURES_REQUIRED "precision-read-${PREC}") + set_tests_properties(test-precision-cleanup-${PREC} PROPERTIES FIXTURES_REQUIRED "precision-cleanup-${PREC}") + set_tests_properties(test-precision-cleanup-${PREC} PROPERTIES FIXTURES_CLEANUP "precision-write-${PREC};precision-read-${PREC}") + + add_test( + NAME + test-lib-precision-write-${PREC} + COMMAND + $ zfpmode=2 prec=${PREC} ofile=test_zfp-lib-prec-${PREC}.h5 + ) + + add_test( + NAME + test-lib-precision-read-${PREC} + COMMAND + $ ret=2 max_reldiff=${DIFF} ifile=test_zfp-lib-prec-${PREC}.h5 + ) + set_tests_properties (test-lib-precision-read-${PREC} PROPERTIES DEPENDS test-lib-precision-write-${PREC}) + + add_test( + NAME + test-lib-precision-cleanup-${PREC} + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-lib-prec-${PREC}.h5 + ) + set_tests_properties (test-lib-precision-cleanup-${PREC} PROPERTIES DEPENDS test-lib-precision-read-${PREC}) + + set_tests_properties(test-lib-precision-write-${PREC} PROPERTIES FIXTURES_SETUP "lib-precision-read-${PREC};lib-precision-cleanup-${PREC}") + set_tests_properties(test-lib-precision-write-${PREC} PROPERTIES FIXTURES_REQUIRED "lib-precision-write-${PREC}") + set_tests_properties(test-lib-precision-read-${PREC} PROPERTIES FIXTURES_REQUIRED "lib-precision-read-${PREC}") + set_tests_properties(test-lib-precision-cleanup-${PREC} PROPERTIES FIXTURES_REQUIRED "lib-precision-cleanup-${PREC}") + set_tests_properties(test-lib-precision-cleanup-${PREC} PROPERTIES FIXTURES_CLEANUP "lib-precision-write-${PREC};lib-precision-read-${PREC}") + + if (FORTRAN_INTERFACE) + add_test( + NAME + test-precision-write-${PREC}-f + COMMAND + $ zfpmode 2 prec ${PREC} ofile test_zfp_fortran-prec-${PREC}.h5 write + ) + + # FIXME: Currently, replaced the difference tests because the relative difference between the two datasets is tricky in this case. + add_test( + NAME + test-precision-diff1-${PREC}-f + COMMAND + echo ${HDF5_DIFF_EXECUTABLE} -v -p 0.00001 test_zfp_fortran-prec-${PREC}.h5 test_zfp_fortran-prec-${PREC}.h5 compressed original + ) + set_property( + TEST + test-precision-diff1-${PREC}-f + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-precision-diff1-${PREC}-f + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-precision-diff1-${PREC}-f PROPERTIES DEPENDS test-precision-write-${PREC}-f) + + add_test( + NAME + test-precision-diff2-${PREC}-f + COMMAND + echo ${HDF5_DIFF_EXECUTABLE} -v -p 0.00001 test_zfp_fortran-prec-${PREC}.h5 test_zfp_fortran-prec-${PREC}.h5 compressed-plugin original + ) + set_property( + TEST + test-precision-diff2-${PREC}-f + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-precision-diff2-${PREC}-f + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-precision-diff2-${PREC}-f PROPERTIES DEPENDS test-precision-diff1-${PREC}-f) + + add_test( + NAME + test-precision-cleanup-${PREC}-f + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp_fortran-prec-${PREC}.h5 + ) + set_tests_properties (test-precision-cleanup-${PREC}-f PROPERTIES DEPENDS test-precision-diff2-${PREC}-f) + + set_tests_properties(test-precision-write-${PREC}-f PROPERTIES FIXTURES_SETUP "precision-diff1-${PREC}-f;precision-diff2-${PREC}-f;precision-cleanup-${PREC}-f") + set_tests_properties(test-precision-write-${PREC}-f PROPERTIES FIXTURES_REQUIRED "precision-write-${PREC}-f") + set_tests_properties(test-precision-diff1-${PREC}-f PROPERTIES FIXTURES_REQUIRED "precision-diff1-${PREC}-f") + set_tests_properties(test-precision-diff2-${PREC}-f PROPERTIES FIXTURES_REQUIRED "precision-diff2-${PREC}-f") + set_tests_properties(test-precision-cleanup-${PREC}-f PROPERTIES FIXTURES_REQUIRED "precision-cleanup-${PREC}-f") + set_tests_properties(test-precision-cleanup-${PREC}-f PROPERTIES FIXTURES_CLEANUP "precision-write-${PREC}-f;precision-diff1-${PREC}-f;precision-diff2-${PREC}-f") + endif () +endforeach () + +# Reversible +add_test( + NAME + test-reversible + COMMAND + $ zfpmode=5 ofile=test_zfp-reversible.h5 +) +set_property( + TEST + test-reversible + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" +) +if (MSVC) + set_property( + TEST + test-reversible + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) +endif () + +add_test( + NAME + test-reversible-diff + COMMAND + $ ret=1 max_absdiff=0 ifile=test_zfp-reversible.h5 +) +set_property( + TEST + test-reversible-diff + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" +) +if (MSVC) + set_property( + TEST + test-reversible-diff + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) +endif () +set_tests_properties (test-reversible-diff PROPERTIES DEPENDS test-reversible) + +add_test( + NAME + test-reversible-cleanup + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-reversible.h5 +) +set_tests_properties (test-reversible-cleanup PROPERTIES DEPENDS test-reversible-diff) + +set_tests_properties(test-reversible PROPERTIES FIXTURES_SETUP "reversible-diff;reversible-cleanup") +set_tests_properties(test-reversible PROPERTIES FIXTURES_REQUIRED "reversible") +set_tests_properties(test-reversible-diff PROPERTIES FIXTURES_REQUIRED "reversible-diff") +set_tests_properties(test-reversible-cleanup PROPERTIES FIXTURES_REQUIRED "reversible-cleanup") +set_tests_properties(test-reversible-cleanup PROPERTIES FIXTURES_CLEANUP "reversible;reversible-diff") + +add_test( + NAME + test-lib-reversible + COMMAND + $ zfpmode=5 ofile=test_zfp-lib-reversible.h5 +) + +add_test( + NAME + test-lib-reversible-diff + COMMAND + $ ret=1 max_absdiff=0 ifile=test_zfp-lib-reversible.h5 +) +set_tests_properties (test-lib-reversible-diff PROPERTIES DEPENDS test-lib-reversible) + +add_test( + NAME + test-lib-reversible-cleanup + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp-lib-reversible.h5 +) +set_tests_properties (test-lib-reversible-cleanup PROPERTIES DEPENDS test-lib-reversible-diff) + +set_tests_properties(test-lib-reversible PROPERTIES FIXTURES_SETUP "lib-reversible-diff;lib-reversible-cleanup") +set_tests_properties(test-lib-reversible PROPERTIES FIXTURES_REQUIRED "lib-reversible") +set_tests_properties(test-lib-reversible-diff PROPERTIES FIXTURES_REQUIRED "lib-reversible-diff") +set_tests_properties(test-lib-reversible-cleanup PROPERTIES FIXTURES_REQUIRED "lib-reversible-cleanup") +set_tests_properties(test-lib-reversible-cleanup PROPERTIES FIXTURES_CLEANUP "lib-reversible;lib-reversible-diff") + +if (FORTRAN_INTERFACE) + add_test( + NAME + test-reversible-write-f + COMMAND + $ zfpmode 5 ofile test_zfp_fortran-reversible.h5 write + ) + + add_test( + NAME + test-reversible-diff1-f + COMMAND + ${HDF5_DIFF_EXECUTABLE} -v -p 0.00001 test_zfp_fortran-reversible.h5 test_zfp_fortran-reversible.h5 compressed original + ) + set_property( + TEST + test-reversible-diff1-f + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-reversible-diff1-f + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-reversible-diff1-f PROPERTIES DEPENDS test-reversible-write-f) + + add_test( + NAME + test-reversible-diff2-f + COMMAND + ${HDF5_DIFF_EXECUTABLE} -v -p 0.00001 test_zfp_fortran-reversible.h5 test_zfp_fortran-reversible.h5 compressed-plugin original + ) + set_property( + TEST + test-reversible-diff2-f + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-reversible-diff2-f + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-reversible-diff2-f PROPERTIES DEPENDS test-reversible-diff1-f) + + add_test( + NAME + test-reversible-cleanup-f + COMMAND + "${CMAKE_COMMAND}" -E remove test_zfp_fortran-reversible.h5 + ) + set_tests_properties (test-reversible-cleanup-f PROPERTIES DEPENDS test-reversible-diff2-f) + + set_tests_properties(test-reversible-write-f PROPERTIES FIXTURES_SETUP "reversible-diff1-f;reversible-diff2-f;reversible-cleanup-f") + set_tests_properties(test-reversible-write-f PROPERTIES FIXTURES_REQUIRED "reversible-write-f") + set_tests_properties(test-reversible-diff1-f PROPERTIES FIXTURES_REQUIRED "reversible-diff1-f") + set_tests_properties(test-reversible-diff2-f PROPERTIES FIXTURES_REQUIRED "reversible-diff2-f") + set_tests_properties(test-reversible-cleanup-f PROPERTIES FIXTURES_REQUIRED "reversible-cleanup-f") + set_tests_properties(test-reversible-cleanup-f PROPERTIES FIXTURES_CLEANUP "reversible-write-f;reversible-diff1-f;reversible-diff2-f") +endif () + +#------------------------------------------------------------------------------# +# Misc tests +#------------------------------------------------------------------------------# +# Integer +add_test(NAME test-lib-int-write COMMAND $ zfpmode=3 doint=1 ofile=test_zfp-int.h5) +add_test(NAME test-lib-int-read COMMAND $ ret=1 max_absdiff=2 ifile=test_zfp-int.h5) +set_tests_properties (test-lib-int-read PROPERTIES DEPENDS test-lib-int-write) +add_test(NAME test-lib-int-cleanup COMMAND "${CMAKE_COMMAND}" -E remove test_zfp-int.h5) +set_tests_properties (test-lib-int-cleanup PROPERTIES DEPENDS test-lib-int-read) + +set_tests_properties(test-lib-int-write PROPERTIES FIXTURES_SETUP "lib-int-read;lib-int-cleanup") +set_tests_properties(test-lib-int-write PROPERTIES FIXTURES_REQUIRED "lib-int-write") +set_tests_properties(test-lib-int-read PROPERTIES FIXTURES_REQUIRED "lib-int-read") +set_tests_properties(test-lib-int-cleanup PROPERTIES FIXTURES_REQUIRED "lib-int-cleanup") +set_tests_properties(test-lib-int-cleanup PROPERTIES FIXTURES_CLEANUP "lib-int-write;lib-int-read") + +# Highd +add_test(NAME test-lib-highd COMMAND $ highd=1 ofile=test_zfp-highd.h5) +add_test(NAME test-lib-highd-cleanup COMMAND "${CMAKE_COMMAND}" -E remove test_zfp-highd.h5) +set_tests_properties (test-lib-highd-cleanup PROPERTIES DEPENDS test-lib-highd) + +set_tests_properties(test-lib-highd PROPERTIES FIXTURES_SETUP "lib-highd-cleanup") +set_tests_properties(test-lib-highd PROPERTIES FIXTURES_REQUIRED "lib-highd") +set_tests_properties(test-lib-highd-cleanup PROPERTIES FIXTURES_REQUIRED "lib-highd-cleanup") +set_tests_properties(test-lib-highd-cleanup PROPERTIES FIXTURES_CLEANUP "lib-highd") + +# Sixd +add_test(NAME test-lib-sixd COMMAND $ sixd=1 ofile=test_zfp-sixd.h5) +add_test(NAME test-lib-sixd-cleanup COMMAND "${CMAKE_COMMAND}" -E remove test_zfp-sixd.h5) +set_tests_properties (test-lib-sixd-cleanup PROPERTIES DEPENDS test-lib-sixd) + +set_tests_properties(test-lib-sixd PROPERTIES FIXTURES_SETUP "lib-sixd-cleanup") +set_tests_properties(test-lib-sixd PROPERTIES FIXTURES_REQUIRED "lib-sixd") +set_tests_properties(test-lib-sixd-cleanup PROPERTIES FIXTURES_REQUIRED "lib-sixd-cleanup") +set_tests_properties(test-lib-sixd-cleanup PROPERTIES FIXTURES_CLEANUP "lib-sixd") + +if (ZFP_CFP_ENABLED) + # zfparr + add_test( + NAME + test-zfparr + COMMAND + $ zfparr=1 rate=10 + ) + + add_test( + NAME + test-zfparr-diff + COMMAND + ${HDF5_DIFF_EXECUTABLE} -v -d 0.01 test_zfp.h5 test_zfp.h5 zfparr_original zfparr_direct + ) + set_property( + TEST + test-zfparr-diff + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-zfparr-diff + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + set_tests_properties (test-zfparr-diff PROPERTIES DEPENDS test-zfparr) + + set_tests_properties(test-zfparr PROPERTIES FIXTURES_SETUP "zfparr-diff") + set_tests_properties(test-zfparr-diff PROPERTIES FIXTURES_REQUIRED "zfparr-diff") +endif () + +# Endian +# FIXME: fails due to datatype difference -> return code 2 +#add_test( +# NAME +# test-endian +# COMMAND +# ${HDF5_DIFF_EXECUTABLE} -v -d 0.00001 ${CMAKE_CURRENT_SOURCE_DIR}/test_zfp_le.h5 ${CMAKE_CURRENT_SOURCE_DIR}/test_zfp_be.h5 compressed compressed +#) +#set_property( +# TEST +# test-endian +# APPEND +# PROPERTY +# ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" +#) +#if (MSVC) +# set_property( +# TEST +# test-endian +# APPEND +# PROPERTY +# ENVIRONMENT +# "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" +# ) +#endif () + +#------------------------------------------------------------------------------# +# Errors tests +#------------------------------------------------------------------------------# +add_executable(test_error test_error.c) +target_compile_definitions(test_write_lib PRIVATE ZFP_LIB_VERSION=0x${ZFP_LIB_VERSION}) +target_link_libraries(test_error h5z_zfp_static) + +add_test(NAME test-error COMMAND $) +add_test(NAME test-error-cleanup COMMAND "${CMAKE_COMMAND}" -E remove test_zfp_errors.h5) +set_tests_properties (test-error-cleanup PROPERTIES DEPENDS test-error) + +set_tests_properties(test-error PROPERTIES FIXTURES_SETUP "error-cleanup") +set_tests_properties(test-error PROPERTIES FIXTURES_REQUIRED "error") +set_tests_properties(test-error-cleanup PROPERTIES FIXTURES_REQUIRED "error-cleanup") +set_tests_properties(test-error-cleanup PROPERTIES FIXTURES_CLEANUP "error") + +#------------------------------------------------------------------------------# +# H5repack tests +#------------------------------------------------------------------------------# +add_executable(print_h5repack_farg print_h5repack_farg.c) +target_link_libraries(print_h5repack_farg h5z_zfp_shared) + +if (NOT MSVC) + add_test( + NAME + test-h5repack + COMMAND + ${HDF5_REPACK_EXECUTABLE} -f UD=32013,0,4,3,0,3539053052,1062232653 + -l X,Y,Z,Indexes:CHUNK=217 + -l Indexes2:CHUNK=1517 + -l Pressure,Pressure2,Pressure3:CHUNK=10x20x5 + -l Pressure_2D:CHUNK=10x20 + -l Stress,Velocity,Stress2,Velocity2,Stress3,Velocity3,VelocityZ,VelocityZ2,VelocityZ3:CHUNK=11x21x1x1 + -l VelocityX_2D:CHUNK=21x31 + -l XY:CHUNK=651x1 + -l XYZ:CHUNK=217x1 + -l XYZ2:CHUNK=1617x1 + -l XYZ3:CHUNK=33x1 + ${CMAKE_CURRENT_SOURCE_DIR}/mesh.h5 mesh_repack.h5 + ) + set_property( + TEST + test-h5repack + APPEND + PROPERTY + ENVIRONMENT "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" + ) + if (MSVC) + set_property( + TEST + test-h5repack + APPEND + PROPERTY + ENVIRONMENT + "PATH=${ESCAPED_PATH}\\;${HDF5_PLUGIN_PATH}" + ) + endif () + + add_test( + NAME + test-h5repack-filesizes + COMMAND + "${CMAKE_COMMAND}" + -D "FILE_ORIGINAL=${CMAKE_CURRENT_SOURCE_DIR}/mesh.h5" + -D "FILE_REPACK=mesh_repack.h5" + -D "RATIO_LIMIT=200" + -P "${CMAKE_CURRENT_SOURCE_DIR}/h5repack-filesizes.cmake" + ) + set_tests_properties (test-h5repack-filesizes PROPERTIES DEPENDS test-h5repack) + + add_test( + NAME + test-h5repack-cleanup + COMMAND + "${CMAKE_COMMAND}" -E rm mesh_repack.h5 + ) + set_tests_properties (test-h5repack-cleanup PROPERTIES DEPENDS test-h5repack-filesizes) + + set_tests_properties(test-h5repack PROPERTIES FIXTURES_SETUP "h5repack-filesizes;h5repack-cleanup") + set_tests_properties(test-h5repack PROPERTIES FIXTURES_REQUIRED "h5repack") + set_tests_properties(test-h5repack-filesizes PROPERTIES FIXTURES_REQUIRED "h5repack-filesizes") + set_tests_properties(test-h5repack-cleanup PROPERTIES FIXTURES_REQUIRED "h5repack-cleanup") + set_tests_properties(test-h5repack-cleanup PROPERTIES FIXTURES_CLEANUP "h5repack;h5repack-filesizes") +endif () + +#------------------------------------------------------------------------------# +# Version compatibility tests +#------------------------------------------------------------------------------# +set(H5FILES ${CMAKE_CURRENT_SOURCE_DIR}/test_zfp_030040.h5 ${CMAKE_CURRENT_SOURCE_DIR}/test_zfp_030235.h5 ${CMAKE_CURRENT_SOURCE_DIR}/test_zfp_110050.h5 ${CMAKE_CURRENT_SOURCE_DIR}/test_zfp_110xxx.h5) +set(RETSTAT FALSE FALSE FALSE TRUE) +list(LENGTH H5FILES NH5FILES) +math(EXPR NH5FILES "${NH5FILES} - 1") +foreach (IFILE RANGE ${NH5FILES}) + list(GET H5FILES ${IFILE} H5FILE) + list(GET RETSTAT ${IFILE} STATUS) + + math(EXPR VERSION_NO "${IFILE} + 1") + add_test(NAME test-version-${VERSION_NO} COMMAND $ ifile=${H5FILE} max_reldiff=0.025 ret=2) + set_tests_properties(test-version-${VERSION_NO} PROPERTIES WILL_FAIL ${STATUS}) +endforeach () + +if (FORTRAN_INTERFACE) + set(H5FILE test_zfp_fortran_version.h5) + math(EXPR VERSION_NO "${NH5FILES} + 2") + add_test(NAME test-version-${VERSION_NO}-input COMMAND $ write dim 1024 zfpmode 1 rate 16 ofile ${H5FILE}) + add_test(NAME test-version-${VERSION_NO} COMMAND $ ifile=${H5FILE} max_reldiff=0.025 ret=2) + add_test(NAME test-version-${VERSION_NO}-cleanup COMMAND "${CMAKE_COMMAND}" -E rm ${H5FILE}) + set_tests_properties(test-version-${VERSION_NO}-input PROPERTIES FIXTURES_SETUP "version;version-cleanup") + set_tests_properties(test-version-${VERSION_NO}-input PROPERTIES FIXTURES_REQUIRED "version-input") + set_tests_properties(test-version-${VERSION_NO} PROPERTIES FIXTURES_REQUIRED "version") + set_tests_properties(test-version-${VERSION_NO}-cleanup PROPERTIES FIXTURES_REQUIRED "version-cleanup") + set_tests_properties(test-version-${VERSION_NO}-cleanup PROPERTIES FIXTURES_CLEANUP "version-input;version") +endif () diff --git a/test/Makefile b/test/Makefile index 46b03dda..2fed9bcb 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,5 +1,11 @@ -include ../config.make +# Include config.make only if we're not making this tool +ifneq ($(strip $(MAKECMDGOALS)),print_h5repack_farg) + ifneq ($(strip $(MAKECMDGOALS)),clean) + include ../config.make + endif +endif +# printf symbols for output of test results NO_COLOR=\033[0m OK_COLOR=\033[32;01m ERROR_COLOR=\033[31;01m @@ -27,7 +33,7 @@ test_write_plugin.o: test_write.c $(CC) -c $< -o $@ -DH5Z_ZFP_USE_PLUGIN -DZFP_LIB_VERSION=0x$(ZFP_LIB_VERSION) $(CFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) test_write_lib.o: test_write.c - $(CC) -c $< -o $@ $(CFLAGS) -DZFP_HAS_CFP=$(ZFP_HAS_CFP) -DZFP_LIB_VERSION=0x$(ZFP_LIB_VERSION) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) + $(CC) -c $< -o $@ $(CFLAGS) -DZFP_HAS_CFP=$(ZFP_HAS_CFP) -DHDF5_HAS_WRITE_CHUNK=$(HDF5_HAS_WRITE_CHUNK) -DZFP_LIB_VERSION=0x$(ZFP_LIB_VERSION) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) test_write_plugin: test_write_plugin.o plugin $(CC) $< -o $@ $(CFLAGS) $(PREPATH)$(HDF5_LIB) $(PREPATH)$(ZFP_LIB) -L$(HDF5_LIB) -L$(ZFP_LIB) -lhdf5 $(ZFP_LIBS) -lm $(LDFLAGS) @@ -42,10 +48,10 @@ test_read_lib.o: test_read.c $(CC) -c $< -o $@ $(CFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) test_read_plugin: test_read_plugin.o plugin - $(CC) $< -o $@ $(PREPATH)$(HDF5_LIB) $(PREPATH)$(ZFP_LIB) -L$(HDF5_LIB) -L$(ZFP_LIB) -lhdf5 $(ZFP_LIBS) $(LDFLAGS) + $(CC) $< -o $@ $(PREPATH)$(HDF5_LIB) $(PREPATH)$(ZFP_LIB) -L$(HDF5_LIB) -L$(ZFP_LIB) -lhdf5 $(ZFP_LIBS) -lm $(LDFLAGS) test_read_lib: test_read_lib.o lib - $(CC) $< -o $@ $(CFLAGS) $(PREPATH)$(HDF5_LIB) $(PREPATH)$(ZFP_LIB) -L../src -L$(HDF5_LIB) -L$(ZFP_LIB) -lh5zzfp -lhdf5 $(ZFP_LIBS) $(LDFLAGS) + $(CC) $< -o $@ $(CFLAGS) $(PREPATH)$(HDF5_LIB) $(PREPATH)$(ZFP_LIB) -L../src -L$(HDF5_LIB) -L$(ZFP_LIB) -lh5zzfp -lhdf5 $(ZFP_LIBS) -lm $(LDFLAGS) test_error.o: test_error.c $(CC) -c $< -o $@ $(CFLAGS) -DZFP_LIB_VERSION=0x$(ZFP_LIB_VERSION) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) @@ -57,7 +63,7 @@ print_h5repack_farg: print_h5repack_farg.o $(CC) $< -o $@ $(LDFLAGS) print_h5repack_farg.o: print_h5repack_farg.c - $(CC) -c $< -o $@ $(CFLAGS) -I$(H5Z_ZFP_BASE) -I$(HDF5_INC) + $(CC) -c $< -o $@ $(CFLAGS) -I../src ifneq ($(FC),) # Fortran Tests [ @@ -346,17 +352,22 @@ test-reversible: plugin test_write_plugin # Uses h5repack to test ZFP filter on float and int datasets in # 1,2,3 and 4 dimensions. Note: need to specify raw cd_values on # command-line to h5repack. We can get these from an invokation -# of test_write which prints them in the header output. The values -# here are for accuracy mode and tolerance of 0.001. +# of print_h5repack_farg. The values here are for accuracy mode +# and tolerance of 0.001. # # A bug-fix patch to h5repack_parse.c is required for this test on # older HDF5 versions (like <= 1.8.10). # -test-h5repack: plugin mesh.h5 patch +# Note use h5repack `-n` option to ensure machine native format +# used when copying datasets to output file. Also, use filter flag +# of 0 in h5repack's UD argument to make the filter mandatory. +# +test-h5repack: plugin mesh.h5 patch print_h5repack_farg @echo " "; \ pad=$$(printf '%*s' "$(padlimit)"); \ pad=$${pad// /.}; \ - x="h5repack -f UD=32013,0,4,3,0,3539053052,1062232653"; \ + rparg=$$(./print_h5repack_farg zfpmode=3 acc=0.001 | grep '^. *-f'); \ + x="h5repack -n $$rparg"; \ printf "$$x"; \ printf ' %*.*s' 0 $$(($(padlength) - $${#x} )) "$$pad"; \ env LD_LIBRARY_PATH=$(HDF5_LIB):$(LD_LIBRARY_PATH) HDF5_PLUGIN_PATH=$(H5Z_ZFP_PLUGIN) $(HDF5_BIN)/$$x \ @@ -408,6 +419,27 @@ test-endian: plugin test_zfp_le.h5 test_zfp_be.h5 fi; \ printf " [$(OK_COLOR)PASSED$(NO_COLOR)] \n" +# Test dumping a file generated on a big endian machine +# The file was generated with: +# yes 12345 | head -n 5000 | h5import /dev/stdin -dims 5000 -type TEXTIN -size 32 -o unpacked.h5 +# h5repack -f UD=32013,0,4,1,0,0,1074921472 unpacked.h5 bigendian.h5 +test-endian1: plugin bigendian.h5 + @echo " "; \ + padlimit=80; \ + padlength=80; \ + pad=$$(printf '%*s' "$${padlimit}"); \ + pad=$${pad// /.}; \ + x="h5dump bigendian.h5"; \ + printf "$$x"; \ + printf ' %*.*s' 0 $$(($${padlength} - $${#x} )) "$$pad"; \ + env LD_LIBRARY_PATH=$(HDF5_LIB):$(LD_LIBRARY_PATH) HDF5_PLUGIN_PATH=$(H5Z_ZFP_PLUGIN) $(HDF5_BIN)/$$x | grep -q '12345, 12345, 12345, 12345,' ; \ + if [[ $$? -ne 0 ]]; then \ + printf " [$(ERROR_COLOR)FAILED$(NO_COLOR)]\n"; \ + touch check-failed; \ + exit 0; \ + fi; \ + printf " [$(OK_COLOR)PASSED$(NO_COLOR)] \n" + # Test the filter as a library rather than as a plugin test-lib-rate: test_write_lib test_read_lib @failed=0; \ @@ -553,7 +585,6 @@ test-int: test_write_lib test_read_lib exit 0; \ elif [[ $$status -eq 2 ]]; then \ printf " [$(SKIP_COLOR)SKIPPED$(NO_COLOR)]\n"; \ - touch check-failed; \ exit 0; \ fi; \ ./test_read_lib ret=1 max_absdiff=2 2>&1 1>/dev/null; \ @@ -590,7 +621,6 @@ test-sixd: test_write_lib status=$$?; \ if [[ $$status -eq 2 ]]; then \ printf " [$(SKIP_COLOR)SKIPPED$(NO_COLOR)]\n"; \ - touch check-failed; \ exit 0; \ elif [[ $$status -ne 0 ]]; then \ printf " [$(ERROR_COLOR)FAILED$(NO_COLOR)]\n"; \ @@ -625,7 +655,6 @@ test-zfparr: test_write_lib status=$$?; \ if [[ $$status -eq 2 ]]; then \ printf " [$(SKIP_COLOR)SKIPPED$(NO_COLOR)]\n"; \ - touch check-failed; \ exit 0; \ elif [[ $$status -ne 0 ]]; then \ printf " [$(ERROR_COLOR)FAILED$(NO_COLOR)]\n"; \ @@ -700,7 +729,7 @@ PLUGIN_CHECK = test-rate test-accuracy test-precision test-h5repack ifneq ($(ZFP_HAS_REVERSIBLE),) PLUGIN_CHECK += test-reversible endif -PLUGIN_CHECK += test-endian +PLUGIN_CHECK += test-endian test-endian1 CHECK = $(LIB_CHECK) $(PLUGIN_CHECK) ifneq ($(FC),) diff --git a/test/bigendian.h5 b/test/bigendian.h5 new file mode 100644 index 00000000..2f12c1de Binary files /dev/null and b/test/bigendian.h5 differ diff --git a/test/h5dump-rate.bsh b/test/h5dump-rate.bsh new file mode 100755 index 00000000..cc53e36d --- /dev/null +++ b/test/h5dump-rate.bsh @@ -0,0 +1,31 @@ +#!/bin/bash +H5DUMP=$1 +FILE=$2 +DSET=$3 +RATE=$4 + +# Checks +if [[ ! -f ${H5DUMP} ]]; then + echo "*** ERROR: The executable h5dump \"${H5DUMP}\" does not exist." + exit 1 +fi + +if [[ ! -f ${FILE} ]]; then + echo "*** ERROR: File \"${FILE}\" does not exist." + exit 1 +fi + + +EXPECTED_RATIO=$(expr 64 / ${RATE}) +ACTUAL_RATIO=`${H5DUMP} -H -d ${DSET} -p ${FILE} | grep COMPRESSION | cut -d':' -f1 | cut -d'(' -f2 | cut -d'.' -f1` +echo "Compression ratio" +echo " h5dump: ${H5DUMP}" +echo " File: ${FILE}" +echo " Dataset: ${DSET}" +echo " Expected: ${EXPECTED_RATIO}" +echo " Actual: ${ACTUAL_RATIO}" +if [[ ${ACTUAL_RATIO} != ${EXPECTED_RATIO} ]]; then + exit 1 +fi + +exit 0 diff --git a/test/h5dump-rate.cmake b/test/h5dump-rate.cmake new file mode 100644 index 00000000..0999ab44 --- /dev/null +++ b/test/h5dump-rate.cmake @@ -0,0 +1,46 @@ +#h5dump-rate.cmake +cmake_policy(SET CMP0007 NEW) + +# arguments checking +if (NOT TEST_PROGRAM) + message (FATAL_ERROR "Require TEST_PROGRAM to be defined") +endif () +if (NOT TEST_FILE) + message (FATAL_ERROR "Require TEST_FILE to be defined") +endif () +if (NOT TEST_DSET) + message (FATAL_ERROR "Require TEST_DSET to be defined") +endif () +if (NOT TEST_RATE) + message (FATAL_ERROR "Require TEST_RATE to be defined") +endif () +if (NOT RATE_START) + message (FATAL_ERROR "Require RATE_START to be defined") +endif () + +math (EXPR EXPECTED_RATIO "64 / ${TEST_RATE}") +# run the test program, capture the stdout/stderr and the result var +execute_process ( + COMMAND ${TEST_PROGRAM} -H -d ${TEST_DSET} -p ${TEST_FILE} + RESULT_VARIABLE TEST_RESULT + OUTPUT_VARIABLE TEST_OUT + ERROR_VARIABLE TEST_ERROR +) +message (STATUS "dump: ${TEST_OUT}") + +# SIZE [0-9]* ([.0-9]*:1 COMPRESSION) +string (REGEX MATCH "SIZE [0-9]* \\(([0-9]*).[0-9]*:1 COMPRESSION\\)" ACTUAL_COMPRESS ${TEST_OUT}) +set (ACTUAL_RATIO ${CMAKE_MATCH_1}) +message (STATUS "Compression ratio") +message (STATUS " File: ${TEST_FILE}") +message (STATUS " Dataset: ${TEST_DSET}") +message (STATUS " Expected: ${EXPECTED_RATIO}") +message (STATUS " Actual: ${ACTUAL_RATIO}") + +if (NOT ${ACTUAL_RATIO} EQUAL ${EXPECTED_RATIO}) + message (FATAL_ERROR "Failed: The ACTUAL_RATIO was DIFFERENT to Expected: ${EXPECTED_RATIO}") +endif () + +# everything went fine... +message (STATUS "RATIO Passed") + diff --git a/test/h5repack-filesizes.bsh b/test/h5repack-filesizes.bsh new file mode 100755 index 00000000..3c8f38d7 --- /dev/null +++ b/test/h5repack-filesizes.bsh @@ -0,0 +1,31 @@ +#!/bin/bash +FILE_ORIGINAL=$1 +FILE_REPACK=$2 + +# Checks +if [[ ! -f ${FILE_ORIGINAL} ]]; then + echo "*** ERROR: Original file \"${FILE_ORIGINAL}\" does not exist." + exit 1 +fi + +if [[ ! -f ${FILE_REPACK} ]]; then + echo "*** ERROR: Repack file \"${FILE_REPACK}\" does not exist." + exit 1 +fi + +ORIG_SIZE=$(ls -l ${FILE_ORIGINAL} | tr -s ' ' | cut -d' ' -f5) +NEW_SIZE=$(ls -l ${FILE_REPACK} | tr -s ' ' | cut -d' ' -f5) +RATIO=`perl -e "printf int($ORIG_SIZE*100/$NEW_SIZE)"` + +echo "Original file" +echo " Name: ${FILE_ORIGINAL}" +echo " Size: ${ORIG_SIZE}" +echo "Repack file" +echo " Name: ${FILE_REPACK}" +echo " Size: ${NEW_SIZE}" +echo " Ratio of the file sizes: ${RATIO}" +if [[ $RATIO -lt 200 ]]; then + exit 1 +fi + +exit 0 diff --git a/test/h5repack-filesizes.cmake b/test/h5repack-filesizes.cmake new file mode 100644 index 00000000..b139ae43 --- /dev/null +++ b/test/h5repack-filesizes.cmake @@ -0,0 +1,33 @@ +#h5repack-filesizes.cmake +cmake_policy(SET CMP0007 NEW) + +# arguments checking +if (NOT FILE_ORIGINAL) + message (FATAL_ERROR "Require FILE_ORIGINAL to be defined") +endif () +if (NOT FILE_REPACK) + message (FATAL_ERROR "Require FILE_REPACK to be defined") +endif () +if (NOT RATIO_LIMIT) + message (FATAL_ERROR "Require RATIO_LIMIT to be defined") +endif () + +file(SIZE ${FILE_ORIGINAL} ORIG_SIZE) +file(SIZE ${FILE_REPACK} NEW_SIZE) +math(EXPR RATIO "(${ORIG_SIZE} * 100) / ${NEW_SIZE}") + +message (STATUS "Original file") +message (STATUS " Name: ${FILE_ORIGINAL}") +message (STATUS " Size: ${ORIG_SIZE}") +message (STATUS "Repack file") +message (STATUS " Name: ${FILE_REPACK}") +message (STATUS " Size: ${NEW_SIZE}") +message (STATUS " Ratio of the file sizes: ${RATIO}") + +if (${RATIO} LESS ${RATIO_LIMIT}) + message (FATAL_ERROR "Failed: The RATIO was LESS ${RATIO_LIMIT}") +endif () + +# everything went fine... +message (STATUS "RATIO Passed") + diff --git a/test/print_h5repack_farg.c b/test/print_h5repack_farg.c index 4afc0844..8467ddd3 100644 --- a/test/print_h5repack_farg.c +++ b/test/print_h5repack_farg.c @@ -12,6 +12,10 @@ This file is part of H5Z-ZFP. Please also read the BSD license #include #include #include +#if defined(_WIN32) || defined(_WIN64) +#define strncasecmp _strnicmp +#define strcasecmp _stricmp +#endif typedef unsigned int uint; @@ -49,16 +53,6 @@ typedef unsigned int uint; printf(" %s%*s\n",tmpstr,60-len,#HELPSTR); \ } - -/* convenience macro to handle errors */ -#define ERROR(FNAME) \ -do { \ - int _errno = errno; \ - fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ - __LINE__, _errno, _errno?strerror(_errno):"ok"); \ - return 1; \ -} while(0) - static void print_cdvals(int zfpmode, double rate, double acc, uint prec, uint minbits, uint maxbits, uint maxprec, int minexp) { @@ -92,8 +86,6 @@ static void print_cdvals(int zfpmode, double rate, double acc, uint prec, int main(int argc, char **argv) { - int i; - /* compression parameters (defaults taken from ZFP header) */ int zfpmode = 1; double rate = 3.5; @@ -106,7 +98,7 @@ int main(int argc, char **argv) int help = 0; /* ZFP filter arguments */ - HANDLE_SEP(Print cdvals for set of ZFP compression paramaters) + HANDLE_SEP(Print cdvals for set of ZFP compression parameters) HANDLE_ARG(zfpmode,(int) strtol(argv[i]+len2,0,10),"%d",set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev)); HANDLE_ARG(rate,(double) strtod(argv[i]+len2,0),"%g",set rate for rate mode of filter); HANDLE_ARG(acc,(double) strtod(argv[i]+len2,0),"%g",set accuracy for accuracy mode of filter); diff --git a/test/test_common.h b/test/test_common.h new file mode 100644 index 00000000..18e4c98f --- /dev/null +++ b/test/test_common.h @@ -0,0 +1,143 @@ +/* +Copyright (c) 2016, Lawrence Livermore National Security, LLC. +Produced at the Lawrence Livermore National Laboratory +Written by Mark C. Miller, miller86@llnl.gov +LLNL-CODE-707197. All rights reserved. + +This file is part of H5Z-ZFP. Please also read the BSD license +https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE +*/ + +#ifndef test_common_H +#define test_common_H + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* ahead of ALL headers to take proper effect */ +#endif + +#ifdef _MSC_VER +#define _CRT_SECURE_NO_DEPRECATE +#define _USE_MATH_DEFINES +#include +#define j0 _j0 +#include +#define strncasecmp _strnicmp +#define strcasecmp _stricmp +#include +#pragma comment(lib, "Shlwapi.lib") +#define strcasestr StrStrIA +#define srandom(X) srand(X) +#define random rand +#define read _read +#define open _open +#define close _close +// strndup() is not available on Windows +char *strndup( const char *s1, size_t n) +{ + char *copy= (char*)malloc( n+1 ); + memcpy( copy, s1, n ); + copy[n] = 0; + return copy; +}; +#else +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include "hdf5.h" + +#define NAME_LEN 256 + +/* convenience macro to handle command-line args and help */ +#define HANDLE_SEP(SEPSTR) \ +{ \ + char tmpstr[64]; \ + int len = snprintf(tmpstr, sizeof(tmpstr), "\n%s...", #SEPSTR);\ + printf(" %*s\n",60-len,tmpstr); \ +} + +#define HANDLE_ARG(A,PARSEA,PRINTA,HELPSTR) \ +{ \ + int i; \ + char tmpstr[64]; \ + int len; \ + int len2 = strlen(#A)+1; \ + for (i = 0; i < argc; i++) \ + { \ + if (!strncmp(argv[i], #A"=", len2)) \ + { \ + A = PARSEA; \ + break; \ + } \ + else if (!strncasecmp(argv[i], "help", 4)) \ + { \ + return 0; \ + } \ + } \ + len = snprintf(tmpstr, sizeof(tmpstr), "%s=" PRINTA, #A, A);\ + printf(" %s%*s\n",tmpstr,60-len,#HELPSTR); \ +} + + +/* convenience macro to handle errors */ +#ifdef _MSC_VER + +#define SET_ERROR(FNAME) \ +do { \ + size_t errmsglen = 94; \ + char errmsg[errmsglen]; \ + strerror_s(errmsg, errmsglen, errno); \ + fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ + __LINE__, errno, errno?errmsg:"ok"); \ + return 1; \ +} while(0) + +#else + +#define SET_ERROR(FNAME) \ +do { \ + int _errno = errno; \ + fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ + __LINE__, _errno, _errno?strerror(_errno):"ok"); \ + return 1; \ +} while(0) + +#endif + +/* Generate a simple, 1D sinusioidal data array with some noise */ +#define TYPINT 1 +#define TYPDBL 2 +static int gen_data(size_t npoints, double noise, double amp, void **_buf, int typ) +{ + size_t i; + double *pdbl = 0; + int *pint = 0; + + /* create data buffer to write */ + if (typ == TYPINT) + pint = (int *) malloc(npoints * sizeof(int)); + else + pdbl = (double *) malloc(npoints * sizeof(double)); + srandom(0xDeadBeef); + for (i = 0; i < npoints; i++) + { + double x = 2 * M_PI * (double) i / (double) (npoints-1); + double n = noise * ((double) random() / ((double)(1<<31)-1) - 0.5); + if (typ == TYPINT) + pint[i] = (int) (amp * (1 + sin(x)) + n); + else + pdbl[i] = (double) (amp * (1 + sin(x)) + n); + } + if (typ == TYPINT) + *_buf = pint; + else + *_buf = pdbl; + return 0; +} +#endif /* test_common_H */ diff --git a/test/test_error.c b/test/test_error.c index 1eff081b..39254e19 100644 --- a/test/test_error.c +++ b/test/test_error.c @@ -5,21 +5,10 @@ Written by Mark C. Miller, miller86@llnl.gov LLNL-CODE-707197. All rights reserved. This file is part of H5Z-ZFP. Please also read the BSD license -https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE +https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE */ -#define _GNU_SOURCE /* ahead of ALL headers to take proper effect */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hdf5.h" +#include "test_common.h" #ifdef H5Z_ZFP_USE_PLUGIN #include "H5Zzfp_plugin.h" @@ -28,90 +17,15 @@ This file is part of H5Z-ZFP. Please also read the BSD license #include "H5Zzfp_props.h" #endif -#define NAME_LEN 256 - -/* convenience macro to handle command-line args and help */ -#define HANDLE_SEP(SEPSTR) \ -{ \ - char tmpstr[64]; \ - int len = snprintf(tmpstr, sizeof(tmpstr), "\n%s...", #SEPSTR);\ - printf(" %*s\n",60-len,tmpstr); \ -} - -#define HANDLE_ARG(A,PARSEA,PRINTA,HELPSTR) \ -{ \ - int i; \ - char tmpstr[64]; \ - int len; \ - int len2 = strlen(#A)+1; \ - for (i = 0; i < argc; i++) \ - { \ - if (!strncmp(argv[i], #A"=", len2)) \ - { \ - A = PARSEA; \ - break; \ - } \ - else if (!strncasecmp(argv[i], "help", 4)) \ - { \ - return 0; \ - } \ - } \ - len = snprintf(tmpstr, sizeof(tmpstr), "%s=" PRINTA, #A, A);\ - printf(" %s%*s\n",tmpstr,60-len,#HELPSTR); \ -} - - -/* convenience macro to handle errors */ -#define ERROR(FNAME) \ -do { \ - int _errno = errno; \ - fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ - __LINE__, _errno, _errno?strerror(_errno):"ok"); \ - return 1; \ -} while(0) - -/* Generate a simple, 1D sinusioidal data array with some noise */ -#define TYPINT 1 -#define TYPDBL 2 -static int gen_data(size_t npoints, double noise, double amp, void **_buf, int typ) -{ - size_t i; - double *pdbl = 0; - int *pint = 0; - - /* create data buffer to write */ - if (typ == TYPINT) - pint = (int *) malloc(npoints * sizeof(int)); - else - pdbl = (double *) malloc(npoints * sizeof(double)); - srandom(0xDeadBeef); - for (i = 0; i < npoints; i++) - { - double x = 2 * M_PI * (double) i / (double) (npoints-1); - double n = noise * ((double) random() / ((double)(1<<31)-1) - 0.5); - if (typ == TYPINT) - pint[i] = (int) (amp * (1 + sin(x)) + n); - else - pdbl[i] = (double) (amp * (1 + sin(x)) + n); - } - if (typ == TYPINT) - *_buf = pint; - else - *_buf = pdbl; - return 0; -} - static hid_t setup_filter(int n, hsize_t *chunk, int zfpmode, - double rate, double acc, uint prec, - uint minbits, uint maxbits, uint maxprec, int minexp) + double rate, double acc, unsigned int prec, + unsigned int minbits, unsigned int maxbits, unsigned int maxprec, int minexp) { hid_t cpid; - unsigned int cd_values[10]; - int i, cd_nelmts = 10; /* setup dataset creation properties */ - if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) ERROR(H5Pcreate); - if (0 > H5Pset_chunk(cpid, n, chunk)) ERROR(H5Pset_chunk); + if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) SET_ERROR(H5Pcreate); + if (0 > H5Pset_chunk(cpid, n, chunk)) SET_ERROR(H5Pset_chunk); /* When filter is used as a library, we need to init it */ H5Z_zfp_initialize(); @@ -135,7 +49,7 @@ static hid_t setup_filter(int n, hsize_t *chunk, int zfpmode, typedef struct client_data {char const *str; int has_str;} client_data_t; static int walk_hdf5_error_stack_cb(unsigned int n, H5E_error_t const *err_desc, void *_cd) -{ +{ client_data_t *cd = (client_data_t *) _cd; if (n > 0) return 0; cd->has_str = strcasestr(err_desc->desc, cd->str) != 0; @@ -154,7 +68,7 @@ static int check_hdf5_error_stack_for_string(char const *str) int main(int argc, char **argv) { - int i, fd, ndiffs; + int i, ndiffs; unsigned corrupt[4] = {0xDeadBeef,0xBabeFace, 0xDeadBabe, 0xBeefFace}; double d = 1.0, *buf = 0, rbuf[DSIZE]; hsize_t chunk[] = {DSIZE,16,16,16,16}; @@ -162,7 +76,7 @@ int main(int argc, char **argv) hsize_t siz; /* HDF5 related variables */ - hid_t fid, tid, dsid, idsid, sid, cpid; + hid_t fid, tid, dsid, sid, cpid; int help = 0; @@ -170,26 +84,26 @@ int main(int argc, char **argv) int zfpmode = H5Z_ZFP_MODE_ACCURACY; double rate = 4; double acc = 0.1; - uint prec = 11; - uint minbits = 0; - uint maxbits = 4171; - uint maxprec = 64; + unsigned int prec = 11; + unsigned int minbits = 0; + unsigned int maxbits = 4171; + unsigned int maxprec = 64; int minexp = -1074; /* ZFP filter arguments */ - HANDLE_SEP(ZFP compression paramaters) - HANDLE_ARG(zfpmode,(int) strtol(argv[i]+len2,0,10),"%d", (1=rate,2=prec,3=acc,4=expert,5=reversible)); + HANDLE_SEP(ZFP compression parameters) + HANDLE_ARG(zfpmode,(int) strtol(argv[i]+len2,0,10),"%d", (1=rate,2=prec,3=acc,4=expert,5=reversible)); HANDLE_ARG(rate,(double) strtod(argv[i]+len2,0),"%g",set rate for rate mode); HANDLE_ARG(acc,(double) strtod(argv[i]+len2,0),"%g",set accuracy for accuracy mode); - HANDLE_ARG(prec,(uint) strtol(argv[i]+len2,0,10),"%u",set precision for precision mode); - HANDLE_ARG(minbits,(uint) strtol(argv[i]+len2,0,10),"%u",set minbits for expert mode); - HANDLE_ARG(maxbits,(uint) strtol(argv[i]+len2,0,10),"%u",set maxbits for expert mode); - HANDLE_ARG(maxprec,(uint) strtol(argv[i]+len2,0,10),"%u",set maxprec for expert mode); + HANDLE_ARG(prec,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set precision for precision mode); + HANDLE_ARG(minbits,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set minbits for expert mode); + HANDLE_ARG(maxbits,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set maxbits for expert mode); + HANDLE_ARG(maxprec,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set maxprec for expert mode); HANDLE_ARG(minexp,(int) strtol(argv[i]+len2,0,10),"%d",set minexp for expert mode); cpid = setup_filter(1, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); - /* Put this after setup_filter to permit printing of otherwise hard to + /* Put this after setup_filter to permit printing of otherwise hard to construct cd_values to facilitate manual invokation of h5repack */ HANDLE_ARG(help,(int)strtol(argv[i]+len2,0,10),"%d",this help message); /* must be last for help to work */ @@ -198,14 +112,14 @@ int main(int argc, char **argv) H5Eset_auto(H5E_DEFAULT, 0, 0); /* setup the 1D data space */ - if (0 > (sid = H5Screate_simple(1, chunk, 0))) ERROR(H5Screate_simple); + if (0 > (sid = H5Screate_simple(1, chunk, 0))) SET_ERROR(H5Screate_simple); /* create HDF5 file */ - if (0 > (fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Fcreate); + if (0 > (fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) SET_ERROR(H5Fcreate); /* test incorrect data type */ tid = H5Tcreate(H5T_STRING, 8); - if (0 <= (dsid = H5Dcreate(fid, "bad_type", tid, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); + if (0 <= (dsid = H5Dcreate(fid, "bad_type", tid, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); #if defined(ZFP_LIB_VERSION) && ZFP_LIB_VERSION<=0x052 assert(check_hdf5_error_stack_for_string("requires datatype class of H5T_FLOAT")); #else @@ -216,13 +130,13 @@ int main(int argc, char **argv) /* test invalid size of data type */ tid = H5Tcopy(H5T_NATIVE_DOUBLE); H5Tset_size(tid, 9); - if (0 <= (dsid = H5Dcreate(fid, "bad_type_size", tid, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); + if (0 <= (dsid = H5Dcreate(fid, "bad_type_size", tid, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); assert(check_hdf5_error_stack_for_string("requires datatype size of 4 or 8")); H5Tclose(tid); /* test invalid chunking on highd data */ cpid = setup_filter(5, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); - if (0 <= (dsid = H5Dcreate(fid, "bad_chunking", H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); + if (0 <= (dsid = H5Dcreate(fid, "bad_chunking", H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); #if defined(ZFP_LIB_VERSION) && ZFP_LIB_VERSION<=0x053 assert(check_hdf5_error_stack_for_string("chunk must have only 1...3 non-unity dimensions")); #else @@ -232,54 +146,57 @@ int main(int argc, char **argv) /* write a compressed dataset to be corrupted later */ cpid = setup_filter(1, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); - if (0 > (dsid = H5Dcreate(fid, "corrupted_data", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) ERROR(H5Dwrite); + if (0 > (dsid = H5Dcreate(fid, "corrupted_data", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); off = 3496; // H5Dget_offset(dsid); siz = H5Dget_storage_size(dsid); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); - if (0 > H5Pclose(cpid)) ERROR(H5Pclose); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); + if (0 > H5Pclose(cpid)) SET_ERROR(H5Pclose); /* write a compressed dataset with some nans and infs */ cpid = setup_filter(1, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); - if (0 > (dsid = H5Dcreate(fid, "nans_and_infs", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); + if (0 > (dsid = H5Dcreate(fid, "nans_and_infs", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); memcpy(rbuf, buf, sizeof(rbuf)); for (i = 7; i < 7+4; i++) rbuf[i] = d/(d-1.0); rbuf[42] = sqrt((double)-1.0); rbuf[42+1] = sqrt((double)-1.0); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) ERROR(H5Dwrite); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); - if (0 > H5Pclose(cpid)) ERROR(H5Pclose); - if (0 > H5Fclose(fid)) ERROR(H5Fclose); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); + if (0 > H5Pclose(cpid)) SET_ERROR(H5Pclose); + if (0 > H5Fclose(fid)) SET_ERROR(H5Fclose); /* Use raw file I/O to corrupt the dataset named corrupted_data */ - fd = open(FNAME, O_RDWR); - pwrite(fd, corrupt, sizeof(corrupt), (off_t) off + (off_t) siz / 3); - close(fd); + FILE *fp; + fp = fopen(FNAME, "rb+"); + if(fp == NULL) SET_ERROR(fopen); + fseek(fp, (off_t) off + (off_t) siz / 3, SEEK_SET); + fwrite(corrupt, 1 , sizeof(corrupt), fp); + fclose(fp); /* Now, open the file with the nans_and_infs and corrupted datasets and try to read them */ - if (0 > (fid = H5Fopen(FNAME, H5F_ACC_RDONLY, H5P_DEFAULT))) ERROR(H5Fopen); - if (0 > (dsid = H5Dopen(fid, "nans_and_infs", H5P_DEFAULT))) ERROR(H5Dopen); - if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) ERROR(H5Dread); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (fid = H5Fopen(FNAME, H5F_ACC_RDONLY, H5P_DEFAULT))) SET_ERROR(H5Fopen); + if (0 > (dsid = H5Dopen(fid, "nans_and_infs", H5P_DEFAULT))) SET_ERROR(H5Dopen); + if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) SET_ERROR(H5Dread); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); for (i = 0, ndiffs = 0; i < DSIZE; i++) { double d = fabs(rbuf[i] - buf[i]); if (d > acc) ndiffs++; } assert(ndiffs == 10); - if (0 > (dsid = H5Dopen(fid, "corrupted_data", H5P_DEFAULT))) ERROR(H5Dopen); - if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) ERROR(H5Dread); + if (0 > (dsid = H5Dopen(fid, "corrupted_data", H5P_DEFAULT))) SET_ERROR(H5Dopen); + if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) SET_ERROR(H5Dread); for (i = 0, ndiffs = 0; i < DSIZE; i++) { double d = fabs(rbuf[i] - buf[i]); if (d > acc) ndiffs++; } assert(ndiffs == 1408); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); free(buf); - if (0 > H5Fclose(fid)) ERROR(H5Fclose); + if (0 > H5Fclose(fid)) SET_ERROR(H5Fclose); H5close(); return 0; diff --git a/test/test_read.c b/test/test_read.c index 19f677a5..a2d2e1e8 100644 --- a/test/test_read.c +++ b/test/test_read.c @@ -62,54 +62,12 @@ LLC, and shall not be used for advertising or product endorsement purposes. */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include - -#include "hdf5.h" +#include "test_common.h" #ifndef H5Z_ZFP_USE_PLUGIN #include "H5Zzfp_lib.h" #endif -#define NAME_LEN 256 - -/* convenience macro to handle command-line args */ -#define HANDLE_ARG(A,PARSEA,PRINTA,HELPSTR) \ -{ \ - int i; \ - char tmpstr[64]; \ - int len; \ - int len2 = strlen(#A)+1; \ - for (i = 0; i < argc; i++) \ - { \ - if (!strncmp(argv[i], #A"=", len2)) \ - { \ - A = PARSEA; \ - break; \ - } \ - else if (!strncmp(#A, "help", 4) && \ - strcasestr(argv[i], "help")) \ - { \ - return 0; \ - } \ - } \ - len = snprintf(tmpstr, sizeof(tmpstr), "%s=" PRINTA, #A, A);\ - printf(" %s%*s\n",tmpstr,60-len,#HELPSTR); \ -} - -/* convenience macro to handle errors */ -#define ERROR(FNAME) \ -do { \ - int _errno = errno; \ - fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ - __LINE__, _errno, _errno?strerror(_errno):"ok"); \ - return 1; \ -} while(0) - int main(int argc, char **argv) { int i, help=0; @@ -133,7 +91,7 @@ int main(int argc, char **argv) int num_reldiffs = 0; int doint = 0; int ret = 0; - + /* file arguments */ strcpy(ifile, "test_zfp.h5"); HANDLE_ARG(ifile,strndup(argv[i]+len2,NAME_LEN), "\"%s\"",set input filename); @@ -148,25 +106,25 @@ int main(int argc, char **argv) #endif /* open the HDF5 file */ - if (0 > (fid = H5Fopen(ifile, H5F_ACC_RDONLY, H5P_DEFAULT))) ERROR(H5Fopen); + if (0 > (fid = H5Fopen(ifile, H5F_ACC_RDONLY, H5P_DEFAULT))) SET_ERROR(H5Fopen); /* read the original dataset */ - if (0 > (dsid = H5Dopen(fid, doint?"int_original":"original", H5P_DEFAULT))) ERROR(H5Dopen); - if (0 > (space_id = H5Dget_space(dsid))) ERROR(H5Dget_space); - if (0 == (npoints = H5Sget_simple_extent_npoints(space_id))) ERROR(H5Sget_simple_extent_npoints); - if (0 > H5Sclose(space_id)) ERROR(H5Sclose); - if (0 == (obuf = (double *) malloc(npoints * sizeof(double)))) ERROR(malloc); - if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, obuf)) ERROR(H5Dread); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); - + if (0 > (dsid = H5Dopen(fid, doint?"int_original":"original", H5P_DEFAULT))) SET_ERROR(H5Dopen); + if (0 > (space_id = H5Dget_space(dsid))) SET_ERROR(H5Dget_space); + if (0 == (npoints = H5Sget_simple_extent_npoints(space_id))) SET_ERROR(H5Sget_simple_extent_npoints); + if (0 > H5Sclose(space_id)) SET_ERROR(H5Sclose); + if (0 == (obuf = (double *) malloc(npoints * sizeof(double)))) SET_ERROR(malloc); + if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, obuf)) SET_ERROR(H5Dread); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); + /* read the compressed dataset */ - if (0 > (dsid = H5Dopen(fid, doint?"int_compressed":"compressed", H5P_DEFAULT))) ERROR(H5Dopen); - if (0 == (cbuf = (double *) malloc(npoints * sizeof(double)))) ERROR(malloc); - if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, cbuf)) ERROR(H5Dread); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dopen(fid, doint?"int_compressed":"compressed", H5P_DEFAULT))) SET_ERROR(H5Dopen); + if (0 == (cbuf = (double *) malloc(npoints * sizeof(double)))) SET_ERROR(malloc); + if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, cbuf)) SET_ERROR(H5Dread); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); /* clean up */ - if (0 > H5Fclose(fid)) ERROR(H5Fclose); + if (0 > H5Fclose(fid)) SET_ERROR(H5Fclose); /* compare original to compressed */ for (i = 0; i < npoints; i++) diff --git a/test/test_rw_fortran.F90 b/test/test_rw_fortran.F90 index c53f9da1..313cd944 100644 --- a/test/test_rw_fortran.F90 +++ b/test/test_rw_fortran.F90 @@ -17,26 +17,26 @@ PROGRAM main INTEGER :: i INTEGER(hsize_t) :: j - ! sinusoid data generation variables + ! sinusoid data generation variables INTEGER(hsize_t) :: npoints ! compression parameters (defaults taken from ZFP header) - integer(C_INT) :: zfpmode = 3 !1=rate, 2=prec, 3=acc, 4=expert + INTEGER(C_INT) :: zfpmode = 3 !1=rate, 2=prec, 3=acc, 4=expert REAL(dp) :: rate = 4_c_double REAL(dp) :: acc = 0_c_double - integer(C_INT) :: prec = 11 - integer(C_INT) :: dim = 0 - integer(C_INT), PARAMETER :: minbits = 0 - integer(C_INT), PARAMETER :: maxbits = 4171 - integer(C_INT), PARAMETER :: maxprec = 64 - integer(C_INT), PARAMETER :: minexp = -1074 + INTEGER(C_INT) :: prec = 11 + INTEGER(C_INT) :: dim = 0 + INTEGER(C_INT), PARAMETER :: minbits = 0 + INTEGER(C_INT), PARAMETER :: maxbits = 4171 + INTEGER(C_INT), PARAMETER :: maxprec = 64 + INTEGER(C_INT), PARAMETER :: minexp = -1074 ! HDF5 related variables INTEGER(hid_t) fid, dsid, sid, cpid, dcpl_id, space_id INTEGER(C_INT), DIMENSION(1:H5Z_ZFP_CD_NELMTS_MEM) :: cd_values INTEGER(C_SIZE_T) :: cd_nelmts = H5Z_ZFP_CD_NELMTS_MEM - ! compressed/uncompressed difference stat variables + ! compressed/uncompressed difference stat variables REAL(dp) :: max_absdiff = 1.e-8_dp REAL(dp) :: max_reldiff = 1.e-8_dp INTEGER(C_INT) :: num_diffs = 0 @@ -67,13 +67,13 @@ LOGICAL FUNCTION real_eq(a,b,ulp) END FUNCTION real_eq END INTERFACE - CHARACTER(LEN=10) :: arg + CHARACTER(LEN=180) :: arg INTEGER :: len LOGICAL :: write_only = .FALSE., avail INTEGER :: config_flag = 0 ! for h5zget_filter_info_f INTEGER :: config_flag_both = 0 ! for h5zget_filter_info_f INTEGER :: nerr = 0 - + DO i = 1, COMMAND_ARGUMENT_COUNT() CALL GET_COMMAND_ARGUMENT(i,arg,len,status) IF (status .NE. 0) THEN @@ -115,6 +115,13 @@ END FUNCTION real_eq STOP 1 END IF READ(arg(1:len), *) prec + ELSE IF (arg(1:len).EQ.'ofile')THEN + CALL GET_COMMAND_ARGUMENT(i+1,arg,len,status) + IF (status .NE. 0) THEN + WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i + STOP 1 + END IF + READ(arg(1:len), *) ofile ELSE IF (arg(1:len).EQ.'write')THEN write_only = .TRUE. @@ -125,14 +132,15 @@ END FUNCTION real_eq PRINT*,"acc - set accuracy for accuracy mode of filter" PRINT*,"prec - set PRECISION for PRECISION mode of zfp filter" PRINT*,"dim - set size of 1D dataset used" + PRINT*,"ofile - set the output file" PRINT*,"write - only write the file" STOP 1 ENDIF - + END DO - ! create data to write if we're not reading from an existing file - + ! create data to write if we're not reading from an existing file + IF (dim .EQ. 0) THEN CALL gen_data(INT(dim1*dim0, c_size_t), noise, amp, wdata) ELSE @@ -146,10 +154,10 @@ END FUNCTION real_eq status = H5Z_zfp_initialize() CALL check("H5Z_zfp_initialize", status, nerr) - ! create HDF5 file + ! create HDF5 file CALL h5fcreate_f(ofile, H5F_ACC_TRUNC_F, fid, status) CALL check("h5fcreate_f", status, nerr) - + ! setup dataset compression via cd_values CALL h5pcreate_f(H5P_DATASET_CREATE_F, cpid, status) @@ -193,7 +201,7 @@ END FUNCTION real_eq CALL check("h5screate_simple_f", status, nerr) END IF - ! write the data WITHOUT compression + ! write the data WITHOUT compression CALL h5dcreate_f(fid, "original", H5T_NATIVE_DOUBLE, sid, dsid, status) CALL check("h5dcreate_f", status, nerr) f_ptr = C_LOC(wdata(1,1)) @@ -206,7 +214,7 @@ END FUNCTION real_eq cd_nelmts = 0 CALL H5Pset_filter_f(cpid, H5Z_FILTER_ZFP, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values, status) CALL check("H5Pset_filter_f", status, nerr) - + CALL h5dcreate_f(fid, "compressed-default", H5T_NATIVE_DOUBLE, sid, dsid, status, dcpl_id=cpid) CALL check("h5dcreate_f", status, nerr) f_ptr = C_LOC(wdata(1,1)) @@ -302,12 +310,12 @@ END FUNCTION real_eq CALL h5fopen_f(ofile, H5F_ACC_RDONLY_F, fid, status) CALL check("h5fopen_f", status, nerr) - - ! read the original dataset + + ! read the original dataset CALL h5dopen_f (fid, "original", dsid, status) CALL check("h5dopen_f", status, nerr) - CALL h5dget_space_f(dsid, space_id,status) + CALL h5dget_space_f(dsid, space_id,status) CALL check("h5dget_space_f", status, nerr) CALL H5Sget_simple_extent_npoints_f(space_id, npoints, status) CALL check("H5Sget_simple_extent_npoints_f", status, nerr) @@ -369,7 +377,7 @@ END FUNCTION real_eq IF(absdiff > max_absdiff) THEN reldiff = 0 IF (obuf(j) .NE. 0) reldiff = absdiff / obuf(j) - + IF (absdiff > max_absdiff) max_absdiff = absdiff IF (reldiff > max_reldiff) max_reldiff = reldiff IF( .NOT.real_eq(obuf(j), cbuf(j), 100) ) THEN diff --git a/test/test_write.c b/test/test_write.c index 776fd808..09ae2d42 100644 --- a/test/test_write.c +++ b/test/test_write.c @@ -5,20 +5,10 @@ Written by Mark C. Miller, miller86@llnl.gov LLNL-CODE-707197. All rights reserved. This file is part of H5Z-ZFP. Please also read the BSD license -https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE +https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hdf5.h" +#include "test_common.h" #ifdef H5Z_ZFP_USE_PLUGIN #include "H5Zzfp_plugin.h" @@ -37,80 +27,6 @@ This file is part of H5Z-ZFP. Please also read the BSD license #endif #endif -#define NAME_LEN 256 - -/* convenience macro to handle command-line args and help */ -#define HANDLE_SEP(SEPSTR) \ -{ \ - char tmpstr[64]; \ - int len = snprintf(tmpstr, sizeof(tmpstr), "\n%s...", #SEPSTR);\ - printf(" %*s\n",60-len,tmpstr); \ -} - -#define HANDLE_ARG(A,PARSEA,PRINTA,HELPSTR) \ -{ \ - int i; \ - char tmpstr[64]; \ - int len; \ - int len2 = strlen(#A)+1; \ - for (i = 0; i < argc; i++) \ - { \ - if (!strncmp(argv[i], #A"=", len2)) \ - { \ - A = PARSEA; \ - break; \ - } \ - else if (!strncmp(#A, "help", 4) && \ - strcasestr(argv[i], "help")) \ - { \ - return 0; \ - } \ - } \ - len = snprintf(tmpstr, sizeof(tmpstr), "%s=" PRINTA, #A, A);\ - printf(" %s%*s\n",tmpstr,60-len,#HELPSTR); \ -} - - -/* convenience macro to handle errors */ -#define ERROR(FNAME) \ -do { \ - int _errno = errno; \ - fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ - __LINE__, _errno, _errno?strerror(_errno):"ok"); \ - return 1; \ -} while(0) - -/* Generate a simple, 1D sinusioidal data array with some noise */ -#define TYPINT 1 -#define TYPDBL 2 -static int gen_data(size_t npoints, double noise, double amp, void **_buf, int typ) -{ - size_t i; - double *pdbl = 0; - int *pint = 0; - - /* create data buffer to write */ - if (typ == TYPINT) - pint = (int *) malloc(npoints * sizeof(int)); - else - pdbl = (double *) malloc(npoints * sizeof(double)); - srandom(0xDeadBeef); - for (i = 0; i < npoints; i++) - { - double x = 2 * M_PI * (double) i / (double) (npoints-1); - double n = noise * ((double) random() / ((double)(1<<31)-1) - 0.5); - if (typ == TYPINT) - pint[i] = (int) (amp * (1 + sin(x)) + n); - else - pdbl[i] = (double) (amp * (1 + sin(x)) + n); - } - if (typ == TYPINT) - *_buf = pint; - else - *_buf = pdbl; - return 0; -} - /* Populate the hyper-dimensional array with samples of a radially symmetric sinc() function but where certain sub-spaces are randomized through dimindx arrays */ static void @@ -195,11 +111,11 @@ static void * gen_random_correlated_array(int typ, int ndims, int const *dims, int nucdims, int const *ucdims) { int i, n; - int nbyt = (int) (typ == TYPINT ? sizeof(int) : sizeof(double)); + int nbyt = (int) (typ == TYPINT ? sizeof(int) : sizeof(double)); unsigned char *buf, *buf0; int m[10]; /* subspace multipliers */ int *dimindx[10]; - + assert(ndims <= 10); /* Set up total size and sub-space multipliers */ @@ -211,7 +127,7 @@ gen_random_correlated_array(int typ, int ndims, int const *dims, int nucdims, in /* allocate buffer of suitable size (doubles or ints) */ buf0 = buf = (unsigned char*) malloc(n * nbyt); - + /* set up dimension identity indexing (e.g. Idx[i]==i) so that we can randomize those dimenions we wish to have UNcorrelated */ for (i = 0; i < ndims; i++) @@ -286,7 +202,7 @@ buffer_time_step(void *tbuf, void *data, int typ, int ndims, int const *dims, in { int i, n; int k = t % 4; - int nbyt = (int) (typ == TYPINT ? sizeof(int) : sizeof(double)); + int nbyt = (int) (typ == TYPINT ? sizeof(int) : sizeof(double)); for (i = 0, n = 1; i < ndims; i++) n *= dims[i]; @@ -299,27 +215,28 @@ static int read_data(char const *fname, size_t npoints, double **_buf) size_t const nbytes = npoints * sizeof(double); int fd; - if (0 > (fd = open(fname, O_RDONLY))) ERROR(open); - if (0 == (*_buf = (double *) malloc(nbytes))) ERROR(malloc); - if (nbytes != read(fd, *_buf, nbytes)) ERROR(read); - if (0 != close(fd)) ERROR(close); + if (0 > (fd = open(fname, O_RDONLY))) SET_ERROR(open); + if (0 == (*_buf = (double *) malloc(nbytes))) SET_ERROR(malloc); + if (nbytes != read(fd, *_buf, nbytes)) SET_ERROR(read); + if (0 != close(fd)) SET_ERROR(close); return 0; } static hid_t setup_filter(int n, hsize_t *chunk, int zfpmode, - double rate, double acc, uint prec, - uint minbits, uint maxbits, uint maxprec, int minexp) + double rate, double acc, unsigned int prec, + unsigned int minbits, unsigned int maxbits, unsigned int maxprec, int minexp) { hid_t cpid; - unsigned int cd_values[10]; - int i; - size_t cd_nelmts = 10; /* setup dataset creation properties */ - if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) ERROR(H5Pcreate); - if (0 > H5Pset_chunk(cpid, n, chunk)) ERROR(H5Pset_chunk); + if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) SET_ERROR(H5Pcreate); + if (0 > H5Pset_chunk(cpid, n, chunk)) SET_ERROR(H5Pset_chunk); #ifdef H5Z_ZFP_USE_PLUGIN + + unsigned int cd_values[10]; + size_t cd_nelmts = 10; + /* setup zfp filter via generic (cd_values) interface */ if (zfpmode == H5Z_ZFP_MODE_RATE) H5Pset_zfp_rate_cdata(rate, cd_nelmts, cd_values); @@ -335,15 +252,15 @@ static hid_t setup_filter(int n, hsize_t *chunk, int zfpmode, cd_nelmts = 0; /* causes default behavior of ZFP library */ /* print cd-values array used for filter */ - printf("%d cd_values= ", (int) cd_nelmts); - for (i = 0; i < (int) cd_nelmts; i++) + printf("\n%d cd_values=", (int) cd_nelmts); + for (int i = 0; i < (int) cd_nelmts; i++) printf("%u,", cd_values[i]); printf("\n"); /* Add filter to the pipeline via generic interface */ - if (0 > H5Pset_filter(cpid, H5Z_FILTER_ZFP, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values)) ERROR(H5Pset_filter); + if (0 > H5Pset_filter(cpid, H5Z_FILTER_ZFP, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values)) SET_ERROR(H5Pset_filter); -#else +#else /* When filter is used as a library, we need to init it */ H5Z_zfp_initialize(); @@ -369,7 +286,6 @@ static hid_t setup_filter(int n, hsize_t *chunk, int zfpmode, int main(int argc, char **argv) { - int i; int retval=0; /* filename variables */ @@ -390,10 +306,10 @@ int main(int argc, char **argv) int zfpmode = H5Z_ZFP_MODE_RATE; double rate = 4; double acc = 0; - uint prec = 11; - uint minbits = 0; - uint maxbits = 4171; - uint maxprec = 64; + unsigned int prec = 11; + unsigned int minbits = 0; + unsigned int maxbits = 4171; + unsigned int maxprec = 64; int minexp = -1074; int *ibuf = 0; double *buf = 0; @@ -408,14 +324,14 @@ int main(int argc, char **argv) HANDLE_ARG(ofile,strndup(argv[i]+len2,NAME_LEN), "\"%s\"",set output filename); /* ZFP filter arguments */ - HANDLE_SEP(ZFP compression paramaters) - HANDLE_ARG(zfpmode,(int) strtol(argv[i]+len2,0,10),"%d", (1=rate,2=prec,3=acc,4=expert,5=reversible)); + HANDLE_SEP(ZFP compression parameters) + HANDLE_ARG(zfpmode,(int) strtol(argv[i]+len2,0,10),"%d", (1=rate,2=prec,3=acc,4=expert,5=reversible)); HANDLE_ARG(rate,(double) strtod(argv[i]+len2,0),"%g",set rate for rate mode); HANDLE_ARG(acc,(double) strtod(argv[i]+len2,0),"%g",set accuracy for accuracy mode); - HANDLE_ARG(prec,(uint) strtol(argv[i]+len2,0,10),"%u",set precision for precision mode); - HANDLE_ARG(minbits,(uint) strtol(argv[i]+len2,0,10),"%u",set minbits for expert mode); - HANDLE_ARG(maxbits,(uint) strtol(argv[i]+len2,0,10),"%u",set maxbits for expert mode); - HANDLE_ARG(maxprec,(uint) strtol(argv[i]+len2,0,10),"%u",set maxprec for expert mode); + HANDLE_ARG(prec,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set precision for precision mode); + HANDLE_ARG(minbits,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set minbits for expert mode); + HANDLE_ARG(maxbits,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set maxbits for expert mode); + HANDLE_ARG(maxprec,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set maxprec for expert mode); HANDLE_ARG(minexp,(int) strtol(argv[i]+len2,0,10),"%d",set minexp for expert mode); /* 1D dataset arguments */ @@ -443,7 +359,7 @@ int main(int argc, char **argv) sixd = 0; #endif -#if defined(ZFP_LIB_VERSION) && ZFP_LIB_VERSION>=0x054 && ZFP_HAS_CFP>0 +#if defined(ZFP_LIB_VERSION) && ZFP_LIB_VERSION>=0x054 && ZFP_HAS_CFP>0 && HDF5_HAS_WRITE_CHUNK>0 HANDLE_ARG(zfparr,(int) strtol(argv[i]+len2,0,10),"%d",run ZFP array case using H5Dwrite_chunk); #else HANDLE_ARG(zfparr,(int) strtol(argv[i]+len2,0,10),"%d",requires ZFP>=0.5.4 with CFP enabled); @@ -453,7 +369,7 @@ int main(int argc, char **argv) cpid = setup_filter(1, &chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); - /* Put this after setup_filter to permit printing of otherwise hard to + /* Put this after setup_filter to permit printing of otherwise hard to construct cd_values to facilitate manual invokation of h5repack */ HANDLE_ARG(help,(int)strtol(argv[i]+len2,0,10),"%d",this help message); /* must be last for help to work */ @@ -469,36 +385,36 @@ int main(int argc, char **argv) gen_data((size_t) npoints, noise*100, amp*1000000, (void**)&ibuf, TYPINT); /* create HDF5 file */ - if (0 > (fid = H5Fcreate(ofile, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Fcreate); + if (0 > (fid = H5Fcreate(ofile, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) SET_ERROR(H5Fcreate); /* setup the 1D data space */ - if (0 > (sid = H5Screate_simple(1, &npoints, 0))) ERROR(H5Screate_simple); + if (0 > (sid = H5Screate_simple(1, &npoints, 0))) SET_ERROR(H5Screate_simple); /* write the data WITHOUT compression */ - if (0 > (dsid = H5Dcreate(fid, "original", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) ERROR(H5Dwrite); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dcreate(fid, "original", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); if (doint) { - if (0 > (idsid = H5Dcreate(fid, "int_original", H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(idsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, ibuf)) ERROR(H5Dwrite); - if (0 > H5Dclose(idsid)) ERROR(H5Dclose); + if (0 > (idsid = H5Dcreate(fid, "int_original", H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(idsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, ibuf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(idsid)) SET_ERROR(H5Dclose); } /* write the data with requested compression */ - if (0 > (dsid = H5Dcreate(fid, "compressed", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) ERROR(H5Dwrite); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dcreate(fid, "compressed", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); if (doint) { - if (0 > (idsid = H5Dcreate(fid, "int_compressed", H5T_NATIVE_INT, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(idsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, ibuf)) ERROR(H5Dwrite); - if (0 > H5Dclose(idsid)) ERROR(H5Dclose); + if (0 > (idsid = H5Dcreate(fid, "int_compressed", H5T_NATIVE_INT, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(idsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, ibuf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(idsid)) SET_ERROR(H5Dclose); } /* clean up from simple tests */ - if (0 > H5Sclose(sid)) ERROR(H5Sclose); - if (0 > H5Pclose(cpid)) ERROR(H5Pclose); + if (0 > H5Sclose(sid)) SET_ERROR(H5Sclose); + if (0 > H5Pclose(cpid)) SET_ERROR(H5Pclose); free(buf); if (ibuf) free(ibuf); @@ -506,7 +422,7 @@ int main(int argc, char **argv) if (highd) { /* dimension indices 0 1 2 3 */ - int fd, dims[] = {256,128,32,16}; + int dims[] = {256,128,32,16}; int ucdims[]={1,3}; /* UNcorrleted dimensions indices */ hsize_t hdims[] = {256,128,32,16}; hsize_t hchunk[] = {256,1,32,1}; @@ -515,21 +431,21 @@ int main(int argc, char **argv) cpid = setup_filter(4, hchunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); - if (0 > (sid = H5Screate_simple(4, hdims, 0))) ERROR(H5Screate_simple); + if (0 > (sid = H5Screate_simple(4, hdims, 0))) SET_ERROR(H5Screate_simple); /* write the data WITHOUT compression */ - if (0 > (dsid = H5Dcreate(fid, "highD_original", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) ERROR(H5Dwrite); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dcreate(fid, "highD_original", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); /* write the data with compression */ - if (0 > (dsid = H5Dcreate(fid, "highD_compressed", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) ERROR(H5Dwrite); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dcreate(fid, "highD_compressed", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); /* clean up from high dimensional test */ - if (0 > H5Sclose(sid)) ERROR(H5Sclose); - if (0 > H5Pclose(cpid)) ERROR(H5Pclose); + if (0 > H5Sclose(sid)) SET_ERROR(H5Sclose); + if (0 > H5Pclose(cpid)) SET_ERROR(H5Pclose); free(buf); } /* End of high dimensional test */ @@ -543,9 +459,9 @@ int main(int argc, char **argv) if (sixd) { void *tbuf; - int t, fd, dims[] = {31,31,31,3,3}; /* a single time instance */ + int t, dims[] = {31,31,31,3,3}; /* a single time instance */ int ucdims[]={3,4}; /* indices of UNcorrleted dimensions in dims (tensor components) */ - hsize_t hdims[] = {31,31,31,3,3,H5S_UNLIMITED}; + hsize_t hdims[] = {31,31,31,3,3,H5S_UNLIMITED}; hsize_t hchunk[] = {31,31,31,1,1,4}; /* 4 non-unity, requires >= ZFP 0.5.4 */ hsize_t hwrite[] = {31,31,31,3,3,4}; /* size/shape of any given H5Dwrite */ @@ -553,10 +469,10 @@ int main(int argc, char **argv) cpid = setup_filter(6, hchunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); /* Create the time-varying, 6D dataset */ - if (0 > (sid = H5Screate_simple(6, hwrite, hdims))) ERROR(H5Screate_simple); - if (0 > (dsid = H5Dcreate(fid, "6D_extendible", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Sclose(sid)) ERROR(H5Sclose); - if (0 > H5Pclose(cpid)) ERROR(H5Pclose); + if (0 > (sid = H5Screate_simple(6, hwrite, hdims))) SET_ERROR(H5Screate_simple); + if (0 > (dsid = H5Dcreate(fid, "6D_extendible", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Sclose(sid)) SET_ERROR(H5Sclose); + if (0 > H5Pclose(cpid)) SET_ERROR(H5Pclose); /* Generate a single buffer which we'll modulate by a time-varying function to represent each timestep */ @@ -567,7 +483,7 @@ int main(int argc, char **argv) tbuf = malloc(31*31*31*3*3*4*sizeof(double)); /* Iterate, writing 9 timesteps by buffering in time 4x. The last - write will contain just one timestep causing ZFP to wind up + write will contain just one timestep causing ZFP to wind up padding all those blocks by 3x along the time dimension. */ for (t = 1; t < 10; t++) { @@ -580,7 +496,7 @@ int main(int argc, char **argv) modulate_by_time(buf, TYPDBL, 5, dims, t); /* Buffer this timestep in memory. Since chunk size in time dimension is 4, - we need to buffer up 4 time steps before we can issue any writes */ + we need to buffer up 4 time steps before we can issue any writes */ buffer_time_step(tbuf, buf, TYPDBL, 5, dims, t); /* If the buffer isn't full, just continue updating it */ @@ -599,26 +515,26 @@ int main(int argc, char **argv) H5Dextend(dsid, hextend); /* Create the memory dataspace */ - if (0 > (msid = H5Screate_simple(6, hwrite, 0))) ERROR(H5Screate_simple); + if (0 > (msid = H5Screate_simple(6, hwrite, 0))) SET_ERROR(H5Screate_simple); /* Get the file dataspace to use for this H5Dwrite call */ - if (0 > (fsid = H5Dget_space(dsid))) ERROR(H5Dget_space); + if (0 > (fsid = H5Dget_space(dsid))) SET_ERROR(H5Dget_space); /* Do a hyperslab selection on the file dataspace for this write*/ - if (0 > H5Sselect_hyperslab(fsid, H5S_SELECT_SET, hstart, 0, hcount, 0)) ERROR(H5Sselect_hyperslab); + if (0 > H5Sselect_hyperslab(fsid, H5S_SELECT_SET, hstart, 0, hcount, 0)) SET_ERROR(H5Sselect_hyperslab); /* Write this iteration to the dataset */ - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, msid, fsid, H5P_DEFAULT, tbuf)) ERROR(H5Dwrite); - if (0 > H5Sclose(msid)) ERROR(H5Sclose); - if (0 > H5Sclose(fsid)) ERROR(H5Sclose); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, msid, fsid, H5P_DEFAULT, tbuf)) SET_ERROR(H5Dwrite); + if (0 > H5Sclose(msid)) SET_ERROR(H5Sclose); + if (0 > H5Sclose(fsid)) SET_ERROR(H5Sclose); } - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); free(buf); free(tbuf); } /* End of 6D Example */ -#if ZFP_HAS_CFP +#if ZFP_HAS_CFP>0 && HDF5_HAS_WRITE_CHUNK>0 /* ZFP Array Example */ if (zfparr>0 && zfpmode==1 && rate>0) { @@ -644,23 +560,23 @@ int main(int argc, char **argv) cpid = setup_filter(2, hchunk_dims, 1, rate, acc, prec, minbits, maxbits, maxprec, minexp); - if (0 > (sid = H5Screate_simple(2, hdims, 0))) ERROR(H5Screate_simple); + if (0 > (sid = H5Screate_simple(2, hdims, 0))) SET_ERROR(H5Screate_simple); /* write the data WITHOUT compression */ - if (0 > (dsid = H5Dcreate(fid, "zfparr_original", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) ERROR(H5Dwrite); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dcreate(fid, "zfparr_original", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); /* write the data with compression via the filter */ - if (0 > (dsid = H5Dcreate(fid, "zfparr_compressed", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) ERROR(H5Dwrite); - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dcreate(fid, "zfparr_compressed", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); /* write the data direct from compressed array using H5Dwrite_chunk calls */ - if (0 > (dsid = H5Dcreate(fid, "zfparr_direct", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite_chunk(dsid, H5P_DEFAULT, 0, hchunk_off, cfp.array2d.compressed_size(origarr), cfp.array2d.compressed_data(origarr))) ERROR(H5Dwrite_chunk); - - if (0 > H5Dclose(dsid)) ERROR(H5Dclose); + if (0 > (dsid = H5Dcreate(fid, "zfparr_direct", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); + if (0 > H5Dwrite_chunk(dsid, H5P_DEFAULT, 0, hchunk_off, cfp.array2d.compressed_size(origarr), cfp.array2d.compressed_data(origarr))) SET_ERROR(H5Dwrite_chunk); + + if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); free(buf); cfp.array2d.dtor(origarr); @@ -668,7 +584,7 @@ int main(int argc, char **argv) /* End of ZFP Array Example */ #endif - if (0 > H5Fclose(fid)) ERROR(H5Fclose); + if (0 > H5Fclose(fid)) SET_ERROR(H5Fclose); free(ifile); free(ofile);