diff --git a/CMakeLists.txt b/CMakeLists.txt index 96852ed60a..5f852a2f98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,10 @@ add_library(netCDF::netcdf ALIAS netcdf) # configure.ac. set(NC_DISPATCH_VERSION 5) +# Version of the plugin path dispatch table. This must match the value in +# configure.ac. +set(NC_PLUGINPATH_DISPATCH_VERSION 1) + # Get system configuration, Use it to determine osname, os release, cpu. These # will be used when committing to CDash. find_program(UNAME NAMES uname) @@ -869,6 +873,7 @@ if(NETCDF_ENABLE_TESTS) ### option(NETCDF_ENABLE_BENCHMARKS "Run benchmark Tests." OFF) + set(BUILD_BENCHMARKS ${NETCDF_ENABLE_BENCHMARKS} CACHE BOOL "alias for NETCDF_ENABLE_BENCHMARKS") ### # End known-failures. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cfcaa3149c..a44b8b1dc0 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #3024](https://github.com/Unidata/netcdf-c/pull/3024) for more information. + ## Known Issue * Parallel operation using `mpich 4.2.0` (the default on `Ubuntu 24.04`) results in 'unexpected results' when running `nc_test4/run_par_test.sh`. This can be fixed by removing `mpich` and associated libraries and development packages and installing `mpich 4.2.2` by hand, or by using `openmpi` provided via `apt`. diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index c21ac3b22b..3313a24bb3 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -226,12 +226,13 @@ if(USE_HDF5) endif(USE_HDF5) ################################ -# Curl Libraryies +# Curl Libraries # Only needed for DAP (DAP2 or DAP4) -# and NCZARR with S3 Support +# and NCZARR S3 support +# and byterange support ################################ -if( (NETCDF_ENABLE_DAP AND (NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT)) OR (NETCDF_ENABLE_NCZARR AND NETCDF_ENABLENCZARR_S3)) +if( NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT OR NETCDF_ENABLE_NCZARR_S3) # See if we have libcurl find_package(CURL) diff --git a/config.h.cmake.in b/config.h.cmake.in index 3e0bff607e..a7f4f43152 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -493,9 +493,12 @@ with zip */ /* Add extra properties to _NCProperties attribute */ #cmakedefine NCPROPERTIES_EXTRA ${NCPROPERTIES_EXTRA} -/* Idspatch table version */ +/* Version for dispatch table version */ #cmakedefine NC_DISPATCH_VERSION ${NC_DISPATCH_VERSION} +/* Version for plugin path dispatch table version */ +#cmakedefine NC_PLUGINPATH_DISPATCH_VERSION ${NC_PLUGINPATH_DISPATCH_VERSION} + /* no IEEE float on this platform */ #cmakedefine NO_IEEE_FLOAT 1 diff --git a/configure.ac b/configure.ac index 33c85470eb..4ca5d82ed3 100644 --- a/configure.ac +++ b/configure.ac @@ -1153,7 +1153,7 @@ AC_ARG_ENABLE([benchmarks], are timed. We use these tests to check netCDF performance.])]) test "x$enable_benchmarks" = xyes || enable_benchmarks=no AC_MSG_RESULT($enable_benchmarks) -if test "x$enable_HDF5" = xno -a "x$enable_benchmarks" = xyes; then +if test "x$enable_hdf5" = xno -a "x$enable_benchmarks" = xyes; then AC_MSG_ERROR([Can't use benchmarks if HDF5 is disabled.]) fi AM_CONDITIONAL(BUILD_BENCHMARKS, [test x$enable_benchmarks = xyes]) @@ -1633,6 +1633,7 @@ has_hdf5_ros3=no if test "x$enable_hdf5" = xyes; then + AC_DEFINE([NETCDF_ENABLE_HDF5], [1], [if true, use HDF5]) AC_DEFINE([USE_HDF5], [1], [if true, use HDF5]) AC_DEFINE([H5_USE_16_API], [1], [use HDF5 1.6 API]) @@ -2258,6 +2259,13 @@ AX_SET_META([NC_HAS_BZ2],[$have_bz2],[yes]) AC_SUBST([NC_DISPATCH_VERSION], [5]) AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.]) +# This is the version of the plugin path dispatch table. +# If the dispatch table is changed, this should be incremented. +# If this is changed, make sure the value in +# CMakeLists.txt also changes to match. +AC_SUBST([NC_PLUGINPATH_DISPATCH_VERSION], [1]) +AC_DEFINE_UNQUOTED([NC_PLUGINPATH_DISPATCH_VERSION], [${NC_PLUGINPATH_DISPATCH_VERSION}], [Plugin Path Dispatch table version.]) + ##### # End netcdf_meta.h definitions. ##### diff --git a/include/Makefile.am b/include/Makefile.am index c20841344c..a9f1127193 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -23,7 +23,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \ hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \ -ncjson.h ncxml.h ncs3sdk.h +ncjson.h ncxml.h ncs3sdk.h ncplugins.h if USE_DAP noinst_HEADERS += ncdap.h diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 5925421d8b..fe2ba7a02c 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -196,6 +196,11 @@ struct NC_HDF5_Filter { unsigned int* params; /**< Params for arbitrary filter. */ }; +/* The type for the NC_FORMATX_NC_HDF5 Global State Object */ +typedef struct GlobalHDF5 { /* libhdf5 dispatcher specific parameters */ + int placeholder; /* ensure a non-empty struct */ +} GlobalHDF5; + int NC4_hdf5_filter_initialize(void); int NC4_hdf5_filter_finalize(void); int NC4_hdf5_filter_remove(NC_VAR_INFO_T* var, unsigned int id); diff --git a/include/nc4internal.h b/include/nc4internal.h index 9a2aac02be..d3b2bceab5 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -107,7 +107,10 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; /* Forward declarations. */ struct NC_GRP_INFO; struct NC_TYPE_INFO; + +/* Opaque */ struct NCRCinfo; +struct NC_PluginPathDispatch; /** * This struct provides indexed Access to Meta-data objects. See the @@ -461,15 +464,10 @@ extern int nc_get_alignment(int* thresholdp, int* alignmentp); /* Begin to collect global state info in one place (more to do) */ typedef struct NCglobalstate { - int initialized; char* tempdir; /* track a usable temp dir */ char* home; /* track $HOME */ char* cwd; /* track getcwd */ struct NCRCinfo* rcinfo; /* Currently only one rc file per session */ - struct GlobalZarr { /* Zarr specific parameters */ - char dimension_separator; - int default_zarrformat; - } zarr; struct GlobalAWS { /* AWS S3 specific parameters/defaults */ char* default_region; char* config_file; @@ -483,6 +481,12 @@ typedef struct NCglobalstate { int alignment; } alignment; struct ChunkCache chunkcache; + /* Global dispatcher and states specific to each dispatcher + and indexed by NC_FORMATX */ + struct FormatXGlobal { + void* state[NC_FORMATX_COUNT]; /* type is opaque (like e.g. file_info_format field) */ + const struct NC_PluginPathDispatch** pluginapi; /*[NC_FORMATX_COUNT];*/ + } formatxstate; } NCglobalstate; extern struct NCglobalstate* NC_getglobalstate(void); diff --git a/include/ncdispatch.h b/include/ncdispatch.h index cee5802b48..7c7a4c3e3e 100644 --- a/include/ncdispatch.h +++ b/include/ncdispatch.h @@ -35,7 +35,7 @@ /* Given a filename, check its magic number */ /* Change magic number size from 4 to 8 to be more precise for HDF5 */ -#define MAGIC_NUMBER_LEN ((size_t)8) +#define MAGIC_NUMBER_LEN ((unsigned long long)8) #define MAGIC_HDF5_FILE 1 #define MAGIC_HDF4_FILE 2 #define MAGIC_CDF1_FILE 1 /* std classic format */ diff --git a/include/ncplugins.h b/include/ncplugins.h new file mode 100644 index 0000000000..c26b8fad54 --- /dev/null +++ b/include/ncplugins.h @@ -0,0 +1,46 @@ +/* +Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata +See COPYRIGHT for license information. +*/ + +/* +Common functionality for plugin paths/ +For internal use only. +*/ + +#ifndef NCPLUGINS_H +#define NCPLUGINS_H + +/* Opaque */ +struct NClist; + +/* Define the plugin path management dispatch table */ + +typedef struct NC_PluginPathDispatch { + int model; /* one of the NC_FORMATX #'s */ + int dispatch_version; + int (*initialize)(void** statep, const struct NClist* initialpaths); + int (*finalize)(void** statep); + int (*read)(void* state, size_t* ndirsp, char** dirs); + int (*write)(void* state, size_t ndirs, char** const dirs); +} NC_PluginPathDispatch; + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Known Plugin Dispatchers */ +#ifdef USE_HDF5 +EXTERNL NC_PluginPathDispatch* NC4_hdf5_pluginpathtable; +#endif +#ifdef NETCDF_ENABLE_NCZARR +EXTERNL NC_PluginPathDispatch* NCZ_pluginpathtable; +#endif + +/* See the file netcdf_aux.h for plugin-related utility functions */ + +#if defined(__cplusplus) +} +#endif + +#endif /*NCPLUGINS_H*/ diff --git a/include/netcdf.h b/include/netcdf.h index ceaeed2b03..675c96302a 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -226,10 +226,13 @@ Use this in mode flags for both nc_create() and nc_open(). */ #define NC_FORMATX_NCZARR (10) #define NC_FORMATX_UNDEFINED (0) - /* To avoid breaking compatibility (such as in the python library), +/* Define the legal NC_FORMATX indices i.e. 0<=index | dirlist + dirlist := dir | dirlist separator dir + separator := ';' | ':' + dir := + +@param pathlist a string encoding a list of directories +@param sep one of ';' | ':' | '\0' where '\0' means use the platform's default separator. +@param ndirsp return the number of directories in dirsp +@param dirsp return a vector of strings representing the directories parsed from pathlist; caller frees +@return ::NC_NOERR + +Note that this function is called twice: first time to get the number of directories +and second to get the directories. + +Author: Dennis Heimbigner +*/ + +EXTERNL int ncaux_plugin_path_parse(const char* pathlist, char sep, size_t* ndirsp, char** dirs); + +/** +Concatenate a vector of directories with the separator between. +This is more-or-less the inverse of the ncaux_plugin_path_parse function + +The resulting string has following syntax: + paths := | dirlist + dirlist := dir | dirlist separator dir + separator := ';' | ':' + dir := + +@param ndirs the number of directories +@param dirsp the directory vector to concatenate +@param sep one of ';', ':', or '\0' +@param catlen length of the cat arg including a nul terminator +@param cat user provided space for holding the concatenation; nul termination guaranteed if catlen > 0. +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments + +Note that this function is called twice: first time to get the expected size of +the concatenated string and second to get the contents of the concatenation. + +Author: Dennis Heimbigner +*/ + +EXTERNL int ncaux_plugin_path_tostring(size_t ndirs, char** const dirs, char sep, size_t* catlen, char* cat); + + +/* +Reclaim a char** object possibly produced by ncaux_plugin_parse function. + +@param veclen the number of entries in vec +@param vec a char** vectore +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments +*/ + +EXTERNL int ncaux_plugin_path_freestringvec(size_t veclen, char** vec); + #if defined(__cplusplus) } #endif diff --git a/include/netcdf_filter.h b/include/netcdf_filter.h index 3c3c88c79b..c46ef654e8 100644 --- a/include/netcdf_filter.h +++ b/include/netcdf_filter.h @@ -12,8 +12,8 @@ #ifndef NETCDF_FILTER_H #define NETCDF_FILTER_H 1 -/* API for libdispatch/dfilter.c -*/ +/**************************************************/ +/* API for libdispatch/dfilter.c */ /* Must match values in */ #ifndef H5Z_FILTER_DEFLATE @@ -110,9 +110,65 @@ EXTERNL int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *leve EXTERNL int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle); EXTERNL int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep); +/* Filter path query/set */ +EXTERNL int nc_filter_path_query(int id); + +/**************************************************/ +/* API for libdispatch/dplugin.c */ + +/* Plugin path functions */ + +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ + +EXTERNL int nc_plugin_path_initialize(void); + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ + +EXTERNL int nc_plugin_path_finalize(void); + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * @param formatx specify which dispatch implementatio to read: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR. + * @param ndirsp return the number of dirs in the internal path list + * @param dirs memory for storing the sequence of directies in the internal path list. + * @return NC_NOERR + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. + * The first time with npaths not NULL and pathlist set to NULL + * to get the size of the path list. + * The second time with pathlist not NULL to get the actual sequence of paths. +*/ + +EXTERNL int nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs); + +/** + * Empty the current internal path sequence + * and replace with the sequence of directories argument. + * + * Using a paths argument of NULL or npaths argument of 0 will clear the set of plugin paths. + * @param formatx specify which dispatch implementation to write: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR. + * @param ndirs length of the dirs argument + * @param dirs to overwrite the current internal path list + * @return NC_NOERR + * @author Dennis Heimbigner +*/ + +EXTERNL int nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs); + #if defined(__cplusplus) } #endif -/**************************************************/ #endif /* NETCDF_FILTER_H */ diff --git a/include/netcdf_json.h b/include/netcdf_json.h index 5d77cadb34..74231a8a8f 100644 --- a/include/netcdf_json.h +++ b/include/netcdf_json.h @@ -1086,7 +1086,6 @@ NCJinsertstring(NCjson* object, const char* key, const char* value) else NCJnewstring(NCJ_STRING,value,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -1100,7 +1099,6 @@ NCJinsertint(NCjson* object, const char* key, long long ivalue) snprintf(digits,sizeof(digits),"%lld",ivalue); NCJnewstring(NCJ_STRING,digits,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -1305,6 +1303,8 @@ netcdf_supresswarnings(void) ignore = (void*)NCJparse; ignore = (void*)NCJdump; ignore = (void*)NCJtotext; + ignore = (void*)NCJinsertstring; + ignore = (void*)NCJinsertint; ignore = ignore; } #endif /*NETCDF_JSON_H*/ diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index 079d009144..f9fe5501cc 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -33,7 +33,7 @@ set_property(SOURCE dinstance_intern.c dinstance.c dvarput.c # Netcdf-4 only functions. Must be defined even if not used target_sources(dispatch PRIVATE - dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c + dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c dplugins.c ) if(BUILD_V2) diff --git a/libdispatch/Makefile.am b/libdispatch/Makefile.am index 9aae84ba0f..bc76c6332e 100644 --- a/libdispatch/Makefile.am +++ b/libdispatch/Makefile.am @@ -34,7 +34,7 @@ libdispatch_la_SOURCES += drc.c # Add functions only found in netCDF-4. # They are always defined, even if they just return an error libdispatch_la_SOURCES += dgroup.c dvlen.c dcompound.c dtype.c denum.c \ -dopaque.c dfilter.c +dopaque.c dfilter.c dplugins.c # Add V2 API convenience library if needed. if BUILD_V2 diff --git a/libdispatch/daux.c b/libdispatch/daux.c index a95dd8977a..374efc60dd 100644 --- a/libdispatch/daux.c +++ b/libdispatch/daux.c @@ -30,6 +30,7 @@ See COPYRIGHT for license information. #include "nclog.h" #include "ncrc.h" #include "netcdf_filter.h" +#include "ncplugins.h" struct NCAUX_FIELD { char* name; @@ -57,6 +58,7 @@ static int computefieldinfo(struct NCAUX_CMPD* cmpd); static int filterspec_cvt(const char* txt, size_t* nparamsp, unsigned int* params); EXTERNL int nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp); +EXTERNL int nc_parse_plugin_pathlist(const char* path0, NClist* dirlist); /**************************************************/ /* @@ -615,7 +617,7 @@ ncaux_h5filterspec_parselist(const char* txt0, int* formatp, size_t* nspecsp, NC done: nullfree(spec); if(vector) { - int i; + size_t i; for(i=0;i 0 */ + if((path = malloc(plen+1+1))==NULL) {stat = NC_ENOMEM; goto done;} + memcpy(path,pathlist0,plen); + path[plen] = '\0'; path[plen+1] = '\0'; /* double null term */ + + for(count=0,p=path;*p;p++) { + if(strchr(seps,*p) == NULL) + continue; /* non-separator */ + else { + *p = '\0'; + count++; + } + } + count++; /* count last piece */ + + /* capture the parsed pieces */ + for(p=path,i=0;i 0) + nclistpush(vec, p); /* use the contents of path */ + p = p+len+1; /* point to next piece */ + } + + if(dirs) { + for(i=0;i 0. +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments + +Note that this function is called twice: first time to get the expected size of +the concatenated string and second to get the contents of the concatenation. +*/ +EXTERNL int +ncaux_plugin_path_tostring(size_t ndirs, char** const dirs, char sep, size_t* catlen, char* cat) +{ + int stat = NC_NOERR; + NCbytes* buf = ncbytesnew(); + size_t i; + + if(sep == '\0') +#ifdef _WIN32 + sep = ';'; +#else + sep = ':'; +#endif + if(cat != NULL) *cat = '\0'; /* Make sure it is nul terminated */ + if(ndirs > 0) { + for(i=0;i0) ncbytesappend(buf,sep); + if(dirs[i] != NULL) ncbytescat(buf,dirs[i]); + } + } + ncbytesnull(buf); + if(cat) + memcpy(cat,ncbytescontents(buf),ncbyteslength(buf)+1); /* include nul termiator */ + if(catlen) *catlen = ncbyteslength(buf)+1; /* overwrite with the true cat length */ + ncbytesfree(buf); + return stat; +} + +/* +Reclaim a char** object possibly produced by ncaux_plugin_parse function. + +@param veclen the number of entries in vec +@param vec a char** vectore +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments +*/ +EXTERNL int +ncaux_plugin_path_freestringvec(size_t veclen, char** vec) +{ + int stat = NC_NOERR; + size_t i; + if(vec == NULL) goto done; + for(i=0;ircinfo = calloc(1,sizeof(struct NCRCinfo)))==NULL) {stat = NC_ENOMEM; goto done;} @@ -176,6 +180,12 @@ NC_createglobalstate(void) if((nc_globalstate->rcinfo->s3profiles = nclistnew())==NULL) {stat = NC_ENOMEM; goto done;} + /* plugin path state */ + if((nc_globalstate->formatxstate.pluginapi = (const NC_PluginPathDispatch**)calloc(NC_FORMATX_COUNT,sizeof(NC_PluginPathDispatch*)))==NULL) + {stat = NC_ENOMEM; goto done;} + memset(nc_globalstate->formatxstate.state,0,NC_FORMATX_COUNT*sizeof(void*)); + if((stat = nc_plugin_path_initialize())) goto done; + /* Get environment variables */ if(getenv(NCRCENVIGNORE) != NULL) nc_globalstate->rcinfo->ignore = 1; @@ -216,6 +226,15 @@ NC_freeglobalstate(void) NC_rcclear(nc_globalstate->rcinfo); free(nc_globalstate->rcinfo); } + { + size_t i; + (void)nc_plugin_path_finalize(); + /* Verify states reclaimed */ + for(i=0;iformatxstate.state[i] == NULL); + memset(nc_globalstate->formatxstate.state,0,NC_FORMATX_COUNT*sizeof(void*)); + nullfree(nc_globalstate->formatxstate.pluginapi); + } free(nc_globalstate); nc_globalstate = NULL; } diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index a381ee0082..938dc00089 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -1284,7 +1284,7 @@ check_file_type(const char *path, int omode, int use_parallel, if((status = openmagic(&magicinfo))) goto done; /* Verify we have a large enough file */ - if(magicinfo.filelen < (unsigned long long)MAGIC_NUMBER_LEN) + if(MAGIC_NUMBER_LEN >= (unsigned long long)magicinfo.filelen) {status = NC_ENOTNC; goto done;} if((status = readmagic(&magicinfo,0L,magic)) != NC_NOERR) { status = NC_ENOTNC; @@ -1306,7 +1306,7 @@ check_file_type(const char *path, int omode, int use_parallel, { size_t pos = 512L; for(;;) { - if((pos+MAGIC_NUMBER_LEN) > magicinfo.filelen) + if((pos+MAGIC_NUMBER_LEN) > (unsigned long long)magicinfo.filelen) {status = NC_ENOTNC; goto done;} if((status = readmagic(&magicinfo,pos,magic)) != NC_NOERR) {status = NC_ENOTNC; goto done; } diff --git a/libdispatch/dinstance.c b/libdispatch/dinstance.c index ad9141474c..cef9b2e893 100644 --- a/libdispatch/dinstance.c +++ b/libdispatch/dinstance.c @@ -484,17 +484,18 @@ static int dump_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset, NCbytes* buf) { int stat = NC_NOERR; - size_t i; + int i; ptrdiff_t saveoffset; int ndims; int dimsizes[NC_MAX_VAR_DIMS]; + int fid; saveoffset = offset->offset; ncbytescat(buf,"<"); /* Get info about each field in turn and dump it */ - for(int fid=0;fid +#include +#include +#ifdef _MSC_VER +#include +#endif + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_aux.h" +#include "ncdispatch.h" +#include "nc4internal.h" +#include "nclog.h" +#include "ncbytes.h" +#include "ncplugins.h" + +/* +Unified plugin related code +*/ +/**************************************************/ +/* Plugin-path API */ + +/* list of environment variables to check for plugin roots */ +#define PLUGIN_ENV "HDF5_PLUGIN_PATH" +#define PLUGIN_DIR_UNIX "/usr/local/hdf5/plugin" +#define PLUGIN_DIR_WIN "%s/hdf5/lib/plugin" +#define WIN32_ROOT_ENV "ALLUSERSPROFILE" + +static int NC_plugin_path_initialized = 0; + +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * + * @return ::NC_NOERR + * + * @author Dennis Heimbigner +*/ + +EXTERNL int +nc_plugin_path_initialize(void) +{ + int stat = NC_NOERR; + struct NCglobalstate* gs = NULL; + char* defaultpluginpath = NULL; + const char* pluginroots = NULL; + NClist* dirs = NULL; + size_t ndirs; + int i; + + if(NC_plugin_path_initialized != 0) goto done; + NC_plugin_path_initialized = 1; + + gs = NC_getglobalstate(); + dirs = nclistnew(); + + /* Setup the plugin path default */ + { +#ifdef _WIN32 + const char* win32_root; + char dfalt[4096]; + win32_root = getenv(WIN32_ROOT_ENV); + if(win32_root != NULL && strlen(win32_root) > 0) { + snprintf(dfalt,sizeof(dfalt),PLUGIN_DIR_WIN,win32_root); + defaultpluginpath = strdup(dfalt); + } +#else /*!_WIN32*/ + defaultpluginpath = strdup(PLUGIN_DIR_UNIX); +#endif + } + + /* Find the plugin directory root(s) */ + pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */ + if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL; + if((stat = ncaux_plugin_path_parse(pluginroots,0,&ndirs,NULL))) goto done; + if(ndirs > 0) { + char** contents; + nclistsetlength(dirs,ndirs); /* may modify contents memory */ + contents = (char**)nclistcontents(dirs); + if((stat = ncaux_plugin_path_parse(pluginroots,0,&ndirs,contents))) goto done; + } + /* Add the default to end of the dirs list if not already there */ + if(defaultpluginpath != NULL && !nclistmatch(dirs,defaultpluginpath,0)) { + nclistpush(dirs,defaultpluginpath); + defaultpluginpath = NULL; + } + + /* Initialize all the plugin path dispatchers and state*/ +#ifdef USE_HDF5 + gs->formatxstate.pluginapi[NC_FORMATX_NC_HDF5] = NC4_hdf5_pluginpathtable; +#endif +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + gs->formatxstate.pluginapi[NC_FORMATX_NCZARR] = NCZ_pluginpathtable; +#endif + /* Initialize all the plugin path dispatcher states */ + for(i=0;iformatxstate.pluginapi[i] != NULL) { + if((stat = gs->formatxstate.pluginapi[i]->initialize(&gs->formatxstate.state[i], dirs))) goto done; + assert(gs->formatxstate.state[i] != NULL); + } + } + +done: + nullfree(defaultpluginpath); + nclistfreeall(dirs); + return NCTHROW(stat); +} + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * + * @return ::NC_NOERR + * + * @author Dennis Heimbigner +*/ + +int +nc_plugin_path_finalize(void) +{ + int stat = NC_NOERR; + struct NCglobalstate* gs = NC_getglobalstate(); + int i; + + if(NC_plugin_path_initialized == 0) goto done; + NC_plugin_path_initialized = 0; + + /* Finalize all the plugin path dispatchers */ + for(i=0;iformatxstate.state[i] != NULL) { + if((stat = gs->formatxstate.pluginapi[i]->finalize(&gs->formatxstate.state[i]))) goto done; + gs->formatxstate.state[i] = NULL; + } + } +done: + return NCTHROW(stat); +} + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * + * @param formatx the dispatcher from which to get the info + * @param ndirsp return the number of paths in the path list + * @param dirs copy the sequence of directories in the path list into this; caller must free the copied strings + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or zero + * + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. The first time + * with ndirsp not NULL and dirs set to NULL to get the size of + * the path list. The second time with dirs not NULL to get the + * actual sequence of paths. +*/ + +EXTERNL int +nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs) +{ + int stat = NC_NOERR; + struct NCglobalstate* gs = NC_getglobalstate(); + + if(formatx < 0 || formatx >= NC_FORMATX_COUNT) {stat = NC_EINVAL; goto done;} + if(!NC_initialized) nc_initialize(); + /* read functions can only apply to specific formatx */ + if(formatx == 0) {stat = NC_EINVAL; goto done;} + + if(gs->formatxstate.pluginapi[formatx] == NULL || gs->formatxstate.state[formatx] == NULL) {stat = NC_EINVAL; goto done;} + if((stat = gs->formatxstate.pluginapi[formatx]->read(gs->formatxstate.state[formatx],ndirsp,dirs))) goto done; +done: + return NCTHROW(stat); +} + +/** + * Empty the current internal path sequence + * and replace with the sequence of directories + * specified in the arguments. + * If ndirs == 0 the path list will be cleared + * + * @param formatx the dispatcher to which to write; zero means all dispatchers + * @param ndirs number of entries in dirs arg + * @param dirs the actual directory paths + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or ndirs > 0 and dirs == NULL + * + * @author Dennis Heimbigner + * + * Note that modifying the plugin paths must be done "atomically". + * That is, in a multi-threaded environment, it is important that + * the sequence of actions involved in setting up the plugin paths + * must be done by a single processor or in some other way as to + * guarantee that two or more processors are not simultaneously + * accessing the plugin path read/write operations. + * + * As an example, assume there exists a mutex lock called PLUGINLOCK. + * Then any processor accessing the plugin paths should operate + * as follows: + *
+ * lock(PLUGINLOCK);
+ * nc_plugin_path_read(...);
+ * 
+ * nc_plugin_path_write(...);
+ * unlock(PLUGINLOCK);
+ * 
+*/ + +EXTERNL int +nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs) +{ + int i,stat = NC_NOERR; + struct NCglobalstate* gs = NC_getglobalstate(); + + if(formatx < 0 || formatx >= NC_FORMATX_COUNT) {stat = NC_EINVAL; goto done;} + if(ndirs > 0 && dirs == NULL) {stat = NC_EINVAL; goto done;} + if(!NC_initialized) nc_initialize(); + /* forall dispatchers */ + for(i=1;iformatxstate.pluginapi[i] == NULL || gs->formatxstate.state[i] == NULL) continue; + if((stat=gs->formatxstate.pluginapi[i]->write(gs->formatxstate.state[i],ndirs,dirs))) goto done; + } + } +done: + return NCTHROW(stat); +} diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 3b81d18c3e..2b81f342bc 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -50,6 +50,8 @@ static int endswith(const char* s, const char* suffix); static void freeentry(struct AWSentry* e); static int awsparse(const char* text, NClist* profiles); +extern void awsprofiles(void); + /**************************************************/ /* Capture environmental Info */ @@ -255,7 +257,7 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) ncurirebuild(newurl); /* return various items */ #ifdef AWSDEBUG - fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",uri->uri,bucket,region); + fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",newurl->uri,bucket,region); #endif if(newurlp) {*newurlp = newurl; newurl = NULL;} if(s3 != NULL) { @@ -507,22 +509,7 @@ NC_aws_load_credentials(NCglobalstate* gstate) gstate->rcinfo->s3profiles = profiles; profiles = NULL; #ifdef AWSDEBUG - {int i,j; - fprintf(stderr,">>> profiles:\n"); - for(i=0;ircinfo->s3profiles);i++) { - struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); - fprintf(stderr," [%s]",p->name); - for(j=0;jentries);j++) { - struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j); - if(strcmp(e->key,"aws_access_key_id") - fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); - else if(strcmp(e->key,"aws_secret_access_key") - fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); - else fprintf(stderr," %s=%s",e->key,e->value); - } - fprintf(stderr,"\n"); - } - } + awsprofiles(); #endif done: @@ -666,7 +653,7 @@ NC_getdefaults3region(NCURI* uri, const char** regionp) if(region == NULL) region = NC_getglobalstate()->aws.default_region; /* Force use of the Amazon default */ #ifdef AWSDEBUG - fprintf(stderr,">>> activeregion = |%s|\n",region)); + fprintf(stderr,">>> activeregion = |%s|\n",region); #endif if(regionp) *regionp = region; return stat; @@ -936,10 +923,52 @@ freeentry(struct AWSentry* e) { if(e) { #ifdef AWSDEBUG -fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value); +fprintf(stderr,">>> freeentry: key=%s value=%s\n",e->key,e->value); #endif nullfree(e->key); nullfree(e->value); nullfree(e); } } + +/* Provide profile-related dumper(s) */ +void +awsdumpprofile(struct AWSprofile* p) +{ + size_t j; + if(p == NULL) { + fprintf(stderr," "); + goto done; + } + fprintf(stderr," [%s]",p->name); + if(p->entries == NULL) { + fprintf(stderr,""); + goto done; + } + for(j=0;jentries);j++) { + struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j); + fprintf(stderr," %s=%s",e->key,e->value); + } +done: + fprintf(stderr,"\n"); +} + +void +awsdumpprofiles(NClist* profiles) +{ + size_t i; + NCglobalstate* gs = NC_getglobalstate(); + for(i=0;ircinfo->s3profiles);i++) { + struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i); + awsdumpprofile(p); + } +} + +void +awsprofiles(void) +{ + NCglobalstate* gs = NC_getglobalstate(); + fprintf(stderr,">>> profiles from global->rcinfo->s3profiles:\n"); + awsdumpprofiles(gs->rcinfo->s3profiles); +} + diff --git a/libdispatch/ncbytes.c b/libdispatch/ncbytes.c index d12c27b432..02ac617075 100644 --- a/libdispatch/ncbytes.c +++ b/libdispatch/ncbytes.c @@ -23,7 +23,7 @@ static int ncbytesfail(void) { fflush(stdout); - fprintf(stderr,"bytebuffer failure\n"); + fprintf(stderr,"NCbytes failure\n"); fflush(stderr); #ifdef NCBYTESDEBUG abort(); diff --git a/libdispatch/ncjson.c b/libdispatch/ncjson.c index 148415666e..4cf21e96d0 100644 --- a/libdispatch/ncjson.c +++ b/libdispatch/ncjson.c @@ -927,7 +927,6 @@ NCJinsertstring(NCjson* object, const char* key, const char* value) else NCJnewstring(NCJ_STRING,value,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -941,7 +940,6 @@ NCJinsertint(NCjson* object, const char* key, long long ivalue) snprintf(digits,sizeof(digits),"%lld",ivalue); NCJnewstring(NCJ_STRING,digits,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -1146,5 +1144,7 @@ netcdf_supresswarnings(void) ignore = (void*)NCJparse; ignore = (void*)NCJdump; ignore = (void*)NCJtotext; + ignore = (void*)NCJinsertstring; + ignore = (void*)NCJinsertint; ignore = ignore; } diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index 56fd3d61ca..7c6adde86e 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -13,6 +13,8 @@ int nclistisnull(void* e) {return e == NULL;} +#define NCLISTDEBUG 1 + #ifndef TRUE #define TRUE 1 #endif @@ -23,6 +25,18 @@ int nclistisnull(void* e) {return e == NULL;} #define DEFAULTALLOC 16 #define ALLOCINCR 16 +static int +nclistfail(void) +{ + fflush(stdout); + fprintf(stderr,"NClist failure\n"); + fflush(stderr); +#ifdef NCLISTDEBUG + abort(); +#endif + return FALSE; +} + NClist* nclistnew(void) { NClist* l; @@ -83,7 +97,7 @@ int nclistsetalloc(NClist* l, size_t sz) { void** newcontent = NULL; - if(l == NULL) return FALSE; + if(l == NULL) return nclistfail(); if(sz <= 0) {sz = (l->length?2*l->length:DEFAULTALLOC);} if(l->alloc >= sz) {return TRUE;} newcontent=(void**)calloc(sz,sizeof(void*)); @@ -99,8 +113,8 @@ nclistsetalloc(NClist* l, size_t sz) int nclistsetlength(NClist* l, size_t newlen) { - if(l == NULL) return FALSE; - if(newlen > l->alloc && !nclistsetalloc(l,newlen)) return FALSE; + if(l == NULL) return nclistfail(); + if(newlen > l->alloc && !nclistsetalloc(l,newlen)) return nclistfail(); if(newlen > l->length) { /* clear any extension */ memset(&l->content[l->length],0,(newlen - l->length)*sizeof(void*)); @@ -112,7 +126,8 @@ nclistsetlength(NClist* l, size_t newlen) void* nclistget(const NClist* l, size_t index) { - if(l == NULL || l->length == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if(l->length == 0) return NULL; if(index >= l->length) return NULL; return l->content[index]; } @@ -123,10 +138,10 @@ nclistget(const NClist* l, size_t index) int nclistset(NClist* l, size_t index, void* elem) { - if(l == NULL) return FALSE; - if(!nclistsetalloc(l,index+1)) return FALSE; + if(l == NULL) return nclistfail(); + if(!nclistsetalloc(l,index+1)) return nclistfail(); if(index >= l->length) { - if(!nclistsetlength(l,index+1)) return FALSE; + if(!nclistsetlength(l,index+1)) return nclistfail(); } l->content[index] = elem; return TRUE; @@ -136,11 +151,13 @@ nclistset(NClist* l, size_t index, void* elem) int nclistinsert(NClist* l, size_t index, void* elem) { - long i; /* do not make unsigned */ - if(l == NULL) return FALSE; - if(index > l->length) return FALSE; + size_t i; + if(l == NULL) return nclistfail(); + if(index > l->length) return nclistfail(); nclistsetalloc(l,0); - for(i=(long)l->length;i>index;i--) l->content[i] = l->content[i-1]; + if(l->length > 0) { + for(i=l->length;i>index;i--) l->content[i] = l->content[i-1]; + } l->content[index] = elem; l->length++; return TRUE; @@ -149,8 +166,10 @@ nclistinsert(NClist* l, size_t index, void* elem) int nclistpush(NClist* l, const void* elem) { - if(l == NULL) return FALSE; + if(l == NULL) return nclistfail(); if(l->length >= l->alloc) nclistsetalloc(l,0); + if(l->content == NULL) + nclistsetalloc(l,0); l->content[l->length] = (void*)elem; l->length++; return TRUE; @@ -159,7 +178,8 @@ nclistpush(NClist* l, const void* elem) void* nclistpop(NClist* l) { - if(l == NULL || l->length == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if(l->length == 0) return NULL; l->length--; return l->content[l->length]; } @@ -167,7 +187,8 @@ nclistpop(NClist* l) void* nclisttop(NClist* l) { - if(l == NULL || l->length == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if(l->length == 0) return NULL; return l->content[l->length - 1]; } @@ -176,7 +197,8 @@ nclistremove(NClist* l, size_t i) { size_t len; void* elem; - if(l == NULL || (len=l->length) == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if((len=l->length) == 0) return NULL; if(i >= len) return NULL; elem = l->content[i]; for(i+=1;icontent[i-1] = l->content[i]; @@ -219,7 +241,8 @@ nclistelemremove(NClist* l, void* elem) size_t len; size_t i; int found = 0; - if(l == NULL || (len=l->length) == 0) return 0; + if(l == NULL) return nclistfail(); + if((len=l->length) == 0) return 0; for(i=0;icontent[i]; if(elem == candidate) { @@ -242,7 +265,8 @@ nclistunique(NClist* l) { size_t i,j,k,len; void** content; - if(l == NULL || l->length == 0) return 1; + if(l == NULL) return nclistfail(); + if(l->length == 0) return 1; len = l->length; content = l->content; for(i=0;ilength == 0) return 1; + if(l == NULL) return nclistfail(); + if(l->length == 0) return 1; nclistpush(l,NULL); nclistsetlength(l,l->length-1); return 1; diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index 0f99dfb473..359ab2f8b3 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -959,11 +959,12 @@ s3objectsinfo(NClist* contents, NClist* keys, NClist* lengths) for(i=0;i #ifdef _WIN32 #include #endif +#include "hdf5internal.h" +#include "hdf5err.h" /* For BAIL2 */ +#include "ncplugins.h" #undef DEBUGH5 diff --git a/libhdf5/hdf5plugins.c b/libhdf5/hdf5plugins.c new file mode 100644 index 0000000000..e1544aff59 --- /dev/null +++ b/libhdf5/hdf5plugins.c @@ -0,0 +1,191 @@ +/* Copyright 2003-2018, University Corporation for Atmospheric + * Research. See the COPYRIGHT file for copying and redistribution + * conditions. + */ +/** + * @file @internal netcdf-4 functions for the plugin list. + * + * @author Dennis Heimbigner + */ + +#include "config.h" +#include +#include +#include "netcdf.h" +#include "ncbytes.h" +#include "hdf5internal.h" +#include "hdf5debug.h" +#include "ncplugins.h" + +#undef TPLUGINS + +/**************************************************/ +/* Forward */ + +static int NC4_hdf5_plugin_path_initialize(void** statep, const NClist* initialpaths); +static int NC4_hdf5_plugin_path_finalize(void** statep); +static int NC4_hdf5_plugin_path_read(void* state, size_t* ndirsp, char** dirs); +static int NC4_hdf5_plugin_path_write(void* state, size_t ndirs, char** const dirs); + +/**************************************************/ +/** + * @file + * @internal + * Internal netcdf hdf5 plugin path functions. + * + * @author Dennis Heimbigner + */ +/**************************************************/ +/* The HDF5 Plugin Path Dispatch table and functions */ + +NC_PluginPathDispatch NC4_hdf5_pluginpathdispatch = { + NC_FORMATX_NC_HDF5, + NC_PLUGINPATH_DISPATCH_VERSION, + NC4_hdf5_plugin_path_initialize, + NC4_hdf5_plugin_path_finalize, + NC4_hdf5_plugin_path_read, + NC4_hdf5_plugin_path_write +}; + +NC_PluginPathDispatch* NC4_hdf5_pluginpathtable = &NC4_hdf5_pluginpathdispatch; + +/**************************************************/ + +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NC4_hdf5_plugin_path_initialize(void** statep, const NClist* initialpaths) +{ + int stat = NC_NOERR; + GlobalHDF5* g5 = NULL; + + NC_UNUSED(initialpaths); /* Let HDF5 do its own thing */ + + assert(statep != NULL); + if(*statep != NULL) goto done; /* already initialized */ + + if((g5 = (GlobalHDF5*)calloc(1,sizeof(GlobalHDF5)))==NULL) {stat = NC_ENOMEM; goto done;} + *statep = (void*)g5; g5 = NULL; +done: + nullfree(g5); + return THROW(stat); +} + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NC4_hdf5_plugin_path_finalize(void** statep) +{ + int stat = NC_NOERR; + GlobalHDF5* g5 = NULL; + + assert(statep != NULL); + if(*statep == NULL) goto done; /* already finalized */ + g5 = (GlobalHDF5*)(*statep); + *statep = NULL; + +done: + nullfree(g5); + return THROW(stat); +} + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * + * @param stat the per-dispatcher global state + * @param ndirsp return the number of paths in the path list + * @param dirs copy the sequence of directories in the path list into this; caller must free the copied strings + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or zero + * + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. The first time + * with ndirsp not NULL and dirs set to NULL to get the size of + * the path list. The second time with dirs not NULL to get the + * actual sequence of paths. +*/ + +static int +NC4_hdf5_plugin_path_read(void* state, size_t* ndirsp, char** dirs) +{ + int stat = NC_NOERR; + herr_t hstat = 0; + unsigned ndirs = 0; + char* dir = NULL; + + if((hstat = H5PLsize(&ndirs))<0) goto done; + if(ndirsp) *ndirsp = ndirs; + if(ndirs > 0 && dirs != NULL) { + unsigned i; + ssize_t dirlen = 0; + for(i=0;i 0 and dirs == NULL + * + * @author Dennis Heimbigner +*/ + +static int +NC4_hdf5_plugin_path_write(void* state, size_t ndirs, char** dirs) +{ + int stat = NC_NOERR; + herr_t hstat = 0; + unsigned hndirs = 0; + + /* Clear the current path list */ + if((hstat = H5PLsize(&hndirs))<0) goto done; + if(hndirs > 0) { + unsigned i; + for(i=0;i 0 && dirs != NULL) { + size_t i; + for(i=0;iformatxstate.state[NC_FORMATX_NCZARR]; + assert(gz != NULL); + + /* Defaults */ + gz->dimension_separator = DFALT_DIM_SEPARATOR; + dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL,NULL); + if(dimsep != NULL) { + /* Verify its value */ + if(dimsep != NULL && strlen(dimsep) == 1 && islegaldimsep(dimsep[0])) + gz->dimension_separator = dimsep[0]; + } stat = NCZ_provenance_init(); - if(stat) ncz_initialized = 1; +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + NCZ_filter_initialize(); +#endif +done: return stat; } @@ -140,9 +162,17 @@ NCZ_initialize(void) int NCZ_finalize(void) { - NCZ_finalize_internal(); + int stat = NC_NOERR; + /* Reclaim global resources */ + ncz_initialized = 0; +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + NCZ_filter_finalize(); +#endif +#ifdef NETCDF_ENABLE_S3 + NCZ_s3finalize(); +#endif NCZ_provenance_finalize(); - return NC_NOERR; + return stat; } static int @@ -198,7 +228,6 @@ NCZ_inq_filter_avail(int ncid, unsigned id) return REPORT(NC_ENOFILTER,"inq_filter_avail"); } - #endif /*NETCDF_ENABLE_NCZARR_FILTERS*/ /**************************************************/ diff --git a/libnczarr/zfile.c b/libnczarr/zfile.c index 2346b4944d..343960da1f 100644 --- a/libnczarr/zfile.c +++ b/libnczarr/zfile.c @@ -294,7 +294,7 @@ NCZ_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp) NC_FILE_INFO_T* file; NC_GRP_INFO_T *grp; int stat = NC_NOERR; - int i; + size_t i; LOG((2, "%s: ncid 0x%x", __func__, ncid)); diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 7481c4ab57..3d0db8d662 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -50,7 +50,8 @@ #include "ncpoco.h" #include "netcdf_filter.h" #include "netcdf_filter_build.h" -#include "netcdf_aux.h" +#include "zfilter.h" +#include "zplugins.h" #if 0 #define DEBUG @@ -63,20 +64,6 @@ #define NULLIFY(x) ((x)?(x):"NULL") -/* Hold the loaded filter plugin information */ -typedef struct NCZ_Plugin { - int incomplete; - struct HDF5API { - const H5Z_class2_t* filter; - NCPSharedLib* hdf5lib; /* source of the filter */ - } hdf5; - struct CodecAPI { - int defaulted; /* codeclib was a defaulting library */ - const NCZ_codec_t* codec; - NCPSharedLib* codeclib; /* of the source codec; null if same as hdf5 */ - } codec; -} NCZ_Plugin; - /* The NC_VAR_INFO_T->filters field is an NClist of this struct */ /* Each filter can have two parts: HDF5 and Codec. @@ -148,14 +135,6 @@ typedef struct NCZ_Filter { /* WARNING: GLOBAL DATA */ /* TODO: move to common global state */ -/* All possible HDF5 filter plugins */ -/* Consider onverting to linked list or hash table or equivalent since very sparse */ -NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX]; -int loaded_plugins_max = -1; - -static NClist* codec_defaults = NULL; /* NClist */ -static NClist* default_libs = NULL; /* NClist; sources of the defaults */ - static int NCZ_filter_initialized = 0; /**************************************************/ @@ -179,61 +158,7 @@ NCJtrace(const NCjson* j) #define SEXISTS(x,p) (((x) && *(x)? (*(x))-> p : "null")) #endif - #if defined(DEBUGF) || defined(DEBUGL) - -const char* -printplugin(const NCZ_Plugin* plugin) -{ - static char plbuf[4096]; - char plbuf2[2000]; - char plbuf1[2000]; - - if(plugin == NULL) return "plugin=NULL"; - plbuf2[0] = '\0'; plbuf1[0] = '\0'; - if(plugin->hdf5.filter) - snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name); - if(plugin->codec.codec) - snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id); - snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2); - return plbuf; -} - -static char* -printparams(size_t nparams, const unsigned* params) -{ - static char ppbuf[4096]; - if(nparams == 0) - snprintf(ppbuf,4096,"{0,%p}",params); - else - snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params)); - return ppbuf; -} - -static char* -printnczparams(const NCZ_Params p) -{ - return printparams(p.nparams,p.params); -} - -static const char* -printcodec(const NCZ_Codec c) -{ - static char pcbuf[4096]; - snprintf(pcbuf,sizeof(pcbuf),"{id=%s codec=%s}", - c.id,NULLIFY(c.codec)); - return pcbuf; -} - -static const char* -printhdf5(const NCZ_HDF5 h) -{ - static char phbuf[4096]; - snprintf(phbuf,sizeof(phbuf),"{id=%u visible=%s working=%s}", - h.id, printnczparams(h.visible), printnczparams(h.working)); - return phbuf; -} - static const char* printfilter(const NCZ_Filter* f) { @@ -249,29 +174,16 @@ printfilter(const NCZ_Filter* f) /* Forward */ -static int NCZ_load_all_plugins(void); -static int NCZ_load_plugin_dir(const char* path); -static int NCZ_load_plugin(const char* path, NCZ_Plugin** plugp); -static int NCZ_unload_plugin(NCZ_Plugin* plugin); -static int NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp); -static int NCZ_plugin_save(int filterid, NCZ_Plugin* p); static int NCZ_filter_free(NCZ_Filter* spec); static int NCZ_filter_hdf5_clear(NCZ_HDF5* spec); static int NCZ_filter_codec_clear(NCZ_Codec* spec); static int NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp); -static int getentries(const char* path, NClist* contents); -static int NCZ_split_plugin_path(const char* path0, NClist* list); - static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter); static int paramnczclone(NCZ_Params* dst, const NCZ_Params* src); static int paramclone(size_t nparams, unsigned** dstp, const unsigned* src); -#ifdef NAMEOPT -static int pluginnamecheck(const char* name); -#endif - /**************************************************/ /** * @file @@ -367,7 +279,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); /* Before anything else, find the matching plugin */ - if((stat = NCZ_plugin_loaded(id,&plugin))) goto done; + if((stat = NCZ_plugin_loaded((size_t)id,&plugin))) goto done; if(plugin == NULL) { stat = THROW(NC_ENOFILTER); goto done; @@ -540,8 +452,6 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, ZTRACE(1,"ncid=%d varid=%d id=%u nparams=%u params=%s",ncid,varid,id,(unsigned)nparams,nczprint_paramvector(nparams,params)); - if((stat = NCZ_filter_initialize())) goto done; - if((stat = NC_check_id(ncid,&nc))) return stat; assert(nc); @@ -607,7 +517,7 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, /* For szip, the pixels_per_block parameter must not be greater * than the number of elements in a chunk of data. */ size_t num_elem = 1; - int d; + size_t d; for (d = 0; d < var->ndims; d++) if (var->dim[d]->len) num_elem *= var->dim[d]->len; @@ -647,9 +557,6 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids assert(h5 && var && var->hdr.id == varid); - /* Make sure all the filters are defined */ - if((stat = NCZ_filter_initialize())) goto done; - flist = var->filters; nfilters = nclistlength(flist); /* including incomplets */ @@ -687,9 +594,6 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, assert(h5 && var && var->hdr.id == varid); - /* Make sure all the plugins are defined */ - if((stat = NCZ_filter_initialize())) goto done; - if((stat = NCZ_filter_lookup(var,id,&spec))) goto done; if(spec != NULL) { #if 0 @@ -734,9 +638,8 @@ NCZ_inq_filter_avail(int ncid, unsigned id) NC_UNUSED(ncid); ZTRACE(1,"ncid=%d id=%u",ncid,id); - if((stat = NCZ_filter_initialize())) goto done; /* Check the available filters list */ - if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done; + if((stat = NCZ_plugin_loaded((size_t)id, &plug))) goto done; if(plug == NULL || plug->incomplete) stat = THROW(NC_ENOFILTER); done: @@ -756,14 +659,11 @@ NCZ_filter_initialize(void) if(NCZ_filter_initialized) goto done; - default_libs = nclistnew(); - codec_defaults = nclistnew(); NCZ_filter_initialized = 1; - memset(loaded_plugins,0,sizeof(loaded_plugins)); + #ifdef NETCDF_ENABLE_NCZARR_FILTERS if((stat = NCZ_load_all_plugins())) goto done; #endif - done: return ZUNTRACE(stat); } @@ -772,75 +672,13 @@ int NCZ_filter_finalize(void) { int stat = NC_NOERR; - int i; - ZTRACE(6,""); if(!NCZ_filter_initialized) goto done; -#ifdef NETCDF_ENABLE_NCZARR_FILTERS - /* Reclaim all loaded filters */ -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: finalize reclaim:\n"); -#endif - for(i=0;i<=loaded_plugins_max;i++) { - if(loaded_plugins[i]) { - NCZ_unload_plugin(loaded_plugins[i]); - loaded_plugins[i] = NULL; - } - } - /* Reclaim the codec defaults */ - if(nclistlength(codec_defaults) > 0) { - for(i=0;i 0) { - for(i=0;i>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l); -#endif - if(l != NULL) (void)ncpsharedlibfree(l); - } - } -#else - memset(loaded_plugins,0,sizeof(loaded_plugins)); -#endif - nclistfree(default_libs); default_libs = NULL; - nclistfree(codec_defaults); codec_defaults = NULL; -done: NCZ_filter_initialized = 0; - return ZUNTRACE(stat); -} -static int -NCZ_plugin_save(int filterid, NCZ_Plugin* p) -{ - int stat = NC_NOERR; - ZTRACE(6,"filterid=%d p=%p",filterid,p); - if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) - {stat = NC_EINVAL; goto done;} - if(filterid > loaded_plugins_max) loaded_plugins_max = filterid; - loaded_plugins[filterid] = p; done: return ZUNTRACE(stat); } -static int -NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp) -{ - int stat = NC_NOERR; - struct NCZ_Plugin* plug = NULL; - ZTRACE(6,"filterid=%d",filterid); - if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) - {stat = NC_EINVAL; goto done;} - if(filterid <= loaded_plugins_max) - plug = loaded_plugins[filterid]; - if(pp) *pp = plug; -done: - return ZUNTRACEX(stat,"plugin=%p",*pp); -} - int NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* chain, size_t inlen, void* indata, size_t* outlenp, void** outdatap, int encode) { @@ -977,7 +815,7 @@ NCZ_filter_jsonize(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Fil int NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex) { - int i,stat = NC_NOERR; + int stat = NC_NOERR; NCZ_Filter* filter = NULL; const NCjson* jvalue = NULL; NCZ_Plugin* plugin = NULL; @@ -1002,13 +840,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j if(NCJunparse(jfilter,0,&codec.codec)<0) {stat = NC_EFILTER; goto done;} /* Find the plugin for this filter */ - for(i=0;i<=loaded_plugins_max;i++) { - if (!loaded_plugins[i]) continue; - if(!loaded_plugins[i] || !loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */ - if(strcmp(NCJstring(jvalue), loaded_plugins[i]->codec.codec->codecid) == 0) - {plugin = loaded_plugins[i]; break;} - } - + if((stat = NCZ_plugin_loaded_byname(NCJstring(jvalue),&plugin))) goto done; /* Will always have a filter; possibly unknown */ if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;} filter->chainindex = chainindex; @@ -1049,561 +881,6 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j return ZUNTRACE(stat); } -/**************************************************/ -/* Filter loading */ - -/* -Get entries in a path that is assumed to be a directory. -*/ - -#ifdef _WIN32 - -static int -getentries(const char* path, NClist* contents) -{ - /* Iterate over the entries in the directory */ - int ret = NC_NOERR; - errno = 0; - WIN32_FIND_DATA FindFileData; - HANDLE dir = NULL; - char* ffpath = NULL; - char* lpath = NULL; - size_t len; - char* d = NULL; - - ZTRACE(6,"path=%s",path); - - /* We need to process the path to make it work with FindFirstFile */ - len = strlen(path); - /* Need to terminate path with '/''*' */ - ffpath = (char*)malloc(len+2+1); - memcpy(ffpath,path,len); - if(path[len-1] != '/') { - ffpath[len] = '/'; - len++; - } - ffpath[len] = '*'; len++; - ffpath[len] = '\0'; - - /* localize it */ - if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done; - dir = FindFirstFile(lpath, &FindFileData); - if(dir == INVALID_HANDLE_VALUE) { - /* Distinquish not-a-directory from no-matching-file */ - switch (GetLastError()) { - case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */ - ret = NC_NOERR; - goto done; - case ERROR_DIRECTORY: /* not a directory */ - default: - ret = NC_EEMPTY; - goto done; - } - } - do { - char* p = NULL; - const char* name = NULL; - name = FindFileData.cFileName; - if(strcmp(name,".")==0 || strcmp(name,"..")==0) - continue; - nclistpush(contents,strdup(name)); - } while(FindNextFile(dir, &FindFileData)); - -done: - if(dir) FindClose(dir); - nullfree(lpath); - nullfree(ffpath); - nullfree(d); - errno = 0; - return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); -} - -#else /* !_WIN32 */ - -int -getentries(const char* path, NClist* contents) -{ - int ret = NC_NOERR; - errno = 0; - DIR* dir = NULL; - - ZTRACE(6,"path=%s",path); - - dir = NCopendir(path); - if(dir == NULL) - {ret = (errno); goto done;} - for(;;) { - const char* name = NULL; - struct dirent* de = NULL; - errno = 0; - de = readdir(dir); - if(de == NULL) - {ret = (errno); goto done;} - if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0) - continue; - name = de->d_name; - nclistpush(contents,strdup(name)); - } -done: - if(dir) NCclosedir(dir); - errno = 0; - return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); -} -#endif /*_WIN32*/ - -static int -NCZ_load_all_plugins(void) -{ - size_t i,j; - int ret = NC_NOERR; - char* pluginroots = NULL; - struct stat buf; - NClist* dirs = nclistnew(); - char* defaultpluginpath = NULL; - - ZTRACE(6,""); - -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); -#endif - - /* Setup the plugin path default */ - { -#ifdef _WIN32 - const char* win32_root; - char dfalt[4096]; - win32_root = getenv(WIN32_ROOT_ENV); - if(win32_root != NULL && strlen(win32_root) > 0) { - snprintf(dfalt,sizeof(dfalt),PLUGIN_DIR_WIN,win32_root); - defaultpluginpath = strdup(dfalt); - } -#else /*!_WIN32*/ - defaultpluginpath = strdup(PLUGIN_DIR_UNIX); -#endif - } - - /* Find the plugin directory root(s) */ - pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */ - if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL; - if(pluginroots == NULL) { - pluginroots = defaultpluginpath; - } - assert(pluginroots != NULL); - ZTRACEMORE(6,"pluginroots=%s",(pluginroots?pluginroots:"null")); - - if((ret = NCZ_split_plugin_path(pluginroots,dirs))) goto done; - - /* Add the default to end of the dirs list if not already there */ - if(!nclistmatch(dirs,defaultpluginpath,0)) { - nclistpush(dirs,defaultpluginpath); - defaultpluginpath = NULL; - } - - for(i=0;icodec->hdf5id); - } - fprintf(stderr,"\n"); - } -#endif - if(nclistlength(codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */ - /* Search the defaults */ - for(j=0;jcodec != NULL) { - const NCZ_codec_t* codec = dfalt->codec; - int hdf5id = codec->hdf5id; - NCZ_Plugin* p = NULL; - if(hdf5id < 0 || hdf5id > loaded_plugins_max) {ret = NC_EFILTER; goto done;} - p = loaded_plugins[hdf5id]; /* get candidate */ - if(p != NULL && p->hdf5.filter != NULL - && p->codec.codec == NULL) { -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path); -#endif - p->codec.codec = codec; - p->codec.codeclib = dfalt->codeclib; - p->codec.defaulted = 1; - } - } - } - } - - /* Mark all plugins for which we do not have both HDF5 and codec */ - { - int i; - NCZ_Plugin* p; - for(i=0;ihdf5.filter == NULL || p->codec.codec == NULL) { - /* mark this entry as incomplete */ - p->incomplete = 1; -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i, - (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); -#endif - } -#ifdef DEBUGL - else - fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i); -#endif - } - } - } - /* Iniitalize all remaining plugins */ - { - int i; - NCZ_Plugin* p; - for(i=0;iincomplete) continue; - if(p->hdf5.filter != NULL && p->codec.codec != NULL) { - if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) - p->codec.codec->NCZ_codec_initialize(); -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); -#endif - } - } - } - } - -done: - nullfree(defaultpluginpath); - nclistfreeall(dirs); - errno = 0; - return ZUNTRACE(ret); -} - -static int -NCZ_split_plugin_path(const char* path0, NClist* list) -{ - int i,stat = NC_NOERR; - char* path = NULL; - char* p; - int count; - size_t plen; -#ifdef _WIN32 - const char* seps = ";"; -#else - const char* seps = ";:"; -#endif - - if(path0 == NULL || path0[0] == '\0') goto done; - plen = strlen(path0); - if((path = malloc(plen+1+1))==NULL) {stat = NC_ENOMEM; goto done;} - memcpy(path,path0,plen); - path[plen] = '\0'; path[plen+1] = '\0'; /* double null term */ - for(count=0,p=path;*p;p++) { - if(strchr(seps,*p) != NULL) {*p = '\0'; count++;} - } - count++; /* for last piece */ - for(p=path,i=0;i 0) - nclistpush(list,strdup(p)); - p = p+len+1; /* point to next piece */ - } - -done: - nullfree(path); - return stat; -} - - -/* Load all the filters within a specified directory */ -static int -NCZ_load_plugin_dir(const char* path) -{ - size_t i; - int stat = NC_NOERR; - size_t pathlen; - NClist* contents = nclistnew(); - char* file = NULL; - - ZTRACE(7,"path=%s",path); - -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_plugin_dir: path=%s\n",path); -#endif - - if(path == NULL) {stat = NC_EINVAL; goto done;} - pathlen = strlen(path); - if(pathlen == 0) {stat = NC_EINVAL; goto done;} - - if((stat = getentries(path,contents))) goto done; - for(i=0;i 0); - nullfree(file); file = NULL; - if((file = (char*)malloc(flen))==NULL) {stat = NC_ENOMEM; goto done;} - file[0] = '\0'; - strlcat(file,path,flen); - strlcat(file,"/",flen); - strlcat(file,name,flen); - /* See if can load the file */ - stat = NCZ_load_plugin(file,&plugin); - switch (stat) { - case NC_NOERR: break; - case NC_ENOFILTER: case NC_ENOTFOUND: - stat = NC_NOERR; - break; /* will cause it to be ignored */ - default: goto done; - } - if(plugin != NULL) { - id = plugin->hdf5.filter->id; - if(loaded_plugins[id] == NULL) { - loaded_plugins[id] = plugin; - if(id > loaded_plugins_max) loaded_plugins_max = id; -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin)); -#endif - } else { -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin duplicate: %s\n",printplugin(plugin)); -#endif - NCZ_unload_plugin(plugin); /* its a duplicate */ - } - } else - stat = NC_NOERR; /*ignore failure */ - } - -done: - nullfree(file); - nclistfreeall(contents); - return ZUNTRACE(stat); -} - -int -loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp) -{ - int stat = NC_NOERR; - int lib_used = 0; - - nclistpush(default_libs,lib); - for(;*cp;cp++) { - struct CodecAPI* c0; -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); -#endif - c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); - if(c0 == NULL) {stat = NC_ENOMEM; goto done;} - c0->codec = *cp; - c0->codeclib = lib; - lib_used = 1; /* remember */ - nclistpush(codec_defaults,c0); c0 = NULL; - } -done: - if(lib_usedp) *lib_usedp = lib_used; - return stat; -} - -static int -NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) -{ - int stat = NC_NOERR; - NCZ_Plugin* plugin = NULL; - const H5Z_class2_t* h5class = NULL; - H5PL_type_t h5type = 0; - const NCZ_codec_t** cp = NULL; - const NCZ_codec_t* codec = NULL; - NCPSharedLib* lib = NULL; - int flags = NCP_GLOBAL; - int h5id = -1; - - assert(path != NULL && strlen(path) > 0 && plugp != NULL); - - ZTRACE(8,"path=%s",path); - - if(plugp) *plugp = NULL; - -#if defined NAMEOPT || defined _WIN32 - /*triage because visual studio does a popup if the file will not load*/ - if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;} -#endif - - /* load the shared library */ - if((stat = ncpsharedlibnew(&lib))) goto done; - if((stat = ncpload(lib,path,flags))) goto done; - -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib); -#endif - - /* See what we have */ - { - const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); - const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info"); - const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); - const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); - - if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) - {stat = THROW(NC_ENOFILTER); goto done;} - - /* We can have cpd or we can have (gpt && gpi && npi) but not both sets */ - if(cpd != NULL) { - cp = (const NCZ_codec_t**)cpd(); - } else {/* cpd => !gpt && !gpi && !npi */ - if(gpt != NULL && gpi != NULL) { /* get HDF5 info */ - h5type = gpt(); - h5class = gpi(); - /* Verify */ - if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} - if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} - } - if(npi != NULL) {/* get Codec info */ - codec = npi(); - /* Verify */ - if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} - if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} - } - } - } - -#ifdef DEBUGL -fprintf(stderr,">>> DEBUGL: load: %s:",path); -if(h5class) fprintf(stderr,">>> %u",(unsigned)h5class->id); -if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid); -fprintf(stderr,">>> \n"); -#endif - - /* Handle defaults separately */ - if(cp != NULL) { - int used = 0; -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); -#endif - if((stat = loadcodecdefaults(path,cp,lib,&used))) goto done; - if(used) lib = NULL; - goto done; - } - - if(h5class != NULL && codec != NULL) { - /* Verify consistency of the HDF5 and the Codec */ - if(h5class->id != codec->hdf5id) goto done; /* ignore */ - } - - /* There are several cases to consider: - 1. This library has both HDF5 API and Codec API => merge - 2. This library has HDF5 API only and Codec API was already found in another library => merge - 3. This library has Codec API only and HDF5 API was already found in another library => merge - */ - - /* Get any previous plugin entry for this id; may be NULL */ - if(h5class != NULL) { - h5id = h5class->id; - if((stat = NCZ_plugin_loaded(h5class->id,&plugin))) goto done; - } else if(codec != NULL) { - h5id = codec->hdf5id; - if((stat = NCZ_plugin_loaded(codec->hdf5id,&plugin))) goto done; - } - - if(plugin == NULL) { - /* create new entry */ - if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;} - } - - /* Fill in the plugin */ - if(h5class != NULL && plugin->hdf5.filter == NULL) { - plugin->hdf5.filter = h5class; - plugin->hdf5.hdf5lib = lib; - lib = NULL; - } - if(codec != NULL && plugin->codec.codec == NULL) { - plugin->codec.codec = codec; - plugin->codec.codeclib = lib; - lib = NULL; - } -#ifdef DEBUGL - if(plugin) - fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin)); -#endif - /* Cleanup */ - if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */ - plugin->codec.codeclib = NULL; - if((stat=NCZ_plugin_save(h5id,plugin))) goto done; - plugin = NULL; - -done: - if(lib) - (void)ncpsharedlibfree(lib); - if(plugin) NCZ_unload_plugin(plugin); - return ZUNTRACEX(stat,"plug=%p",*plugp); -} - -static int -NCZ_unload_plugin(NCZ_Plugin* plugin) -{ - ZTRACE(9,"plugin=%p",plugin); - - if(plugin) { -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin)); -#endif - if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize) - plugin->codec.codec->NCZ_codec_finalize(); - if(plugin->hdf5.filter != NULL) loaded_plugins[plugin->hdf5.filter->id] = NULL; - if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib); - if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); -memset(plugin,0,sizeof(NCZ_Plugin)); - free(plugin); - } - return ZUNTRACE(NC_NOERR); -} - -#ifdef NAMEOPT -static int -pluginnamecheck(const char* name) -{ - size_t count,len; - long i; - const char* p; - if(name == NULL) return 0; - /* get basename */ - p = strrchr(name,'/'); - if(p != NULL) name = (p+1); - len = strlen(name); - if(len == 0) return 0; - i = (long)(len-1); - count = 1; - p = name+i; - for(;i>=0;i--,count++,p--) { - char c = *p; - if(c == '/') break; - if(c == '.') { - if(count >= 3 && memcmp(p,".so",3)==0) return 1; - if(count >= 4 && memcmp(p,".dll",4)==0) return 1; - if(count >= 6 && memcmp(p,".dylib",6)==0) return 1; - } - } - return 0; -} -#endif - /**************************************************/ /* _Codecs attribute */ diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 4a9ae0b101..31b64d65f5 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -19,24 +19,18 @@ /*Mnemonic*/ #define ENCODING 1 -/* list of environment variables to check for plugin roots */ -#define PLUGIN_ENV "HDF5_PLUGIN_PATH" -#define PLUGIN_DIR_UNIX "/usr/local/hdf5/plugin" -#define PLUGIN_DIR_WIN "%s/hdf5/lib/plugin" -#define WIN32_ROOT_ENV "ALLUSERSPROFILE" - /* Opaque */ struct NCZ_Filter; -int NCZ_filter_initialize(void); -int NCZ_filter_finalize(void); -int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); -int NCZ_filter_setup(NC_VAR_INFO_T* var); -int NCZ_filter_freelists(NC_VAR_INFO_T* var); -int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar); -int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode); -int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); -int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex); -int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); +EXTERNL int NCZ_filter_initialize(void); +EXTERNL int NCZ_filter_finalize(void); +EXTERNL int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); +EXTERNL int NCZ_filter_setup(NC_VAR_INFO_T* var); +EXTERNL int NCZ_filter_freelists(NC_VAR_INFO_T* var); +EXTERNL int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar); +EXTERNL int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode); +EXTERNL int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); +EXTERNL int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex); +EXTERNL int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); #endif /*ZFILTER_H*/ diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 81c6fe0ee4..a654041011 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -46,51 +46,6 @@ set_auto(void* func, void *client_data) } #endif -/** - * @internal Provide a function to do any necessary initialization of - * the ZARR library. - */ -int -NCZ_initialize_internal(void) -{ - int stat = NC_NOERR; - char* dimsep = NULL; - NCglobalstate* ngs = NULL; - - ncz_initialized = 1; - ngs = NC_getglobalstate(); - if(ngs != NULL) { - /* Defaults */ - ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR; - dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL,NULL); - if(dimsep != NULL) { - /* Verify its value */ - if(dimsep != NULL && strlen(dimsep) == 1 && islegaldimsep(dimsep[0])) - ngs->zarr.dimension_separator = dimsep[0]; - } - } - - return stat; -} - -/** - * @internal Provide a function to do any necessary finalization of - * the ZARR library. - */ -int -NCZ_finalize_internal(void) -{ - /* Reclaim global resources */ - ncz_initialized = 0; -#ifdef NETCDF_ENABLE_NCZARR_FILTERS - NCZ_filter_finalize(); -#endif -#ifdef NETCDF_ENABLE_S3 - NCZ_s3finalize(); -#endif - return NC_NOERR; -} - /** * @internal Given a varid, return the maximum length of a dimension * using dimid. @@ -113,7 +68,7 @@ find_var_dim_max_length(NC_GRP_INFO_T *grp, int varid, int dimid, *maxlen = 0; /* Find this var. */ - var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); + var = (NC_VAR_INFO_T*)ncindexith(grp->vars,(size_t)varid); if (!var) return NC_ENOTVAR; assert(var->hdr.id == varid); @@ -230,7 +185,7 @@ ncz_find_dim_len(NC_GRP_INFO_T *grp, int dimid, size_t **len) { NC_VAR_INFO_T *var; int retval; - int i; + size_t i; assert(grp && len); LOG((3, "%s: grp->name %s dimid %d", __func__, grp->hdr.name, dimid)); @@ -382,7 +337,7 @@ static int close_dims(NC_GRP_INFO_T *grp) { NC_DIM_INFO_T *dim; - int i; + size_t i; for (i = 0; i < ncindexsize(grp->dim); i++) { @@ -417,7 +372,7 @@ close_dims(NC_GRP_INFO_T *grp) static int close_types(NC_GRP_INFO_T *grp) { - int i; + size_t i; for (i = 0; i < ncindexsize(grp->type); i++) { @@ -456,7 +411,7 @@ close_types(NC_GRP_INFO_T *grp) static int ncz_rec_grp_NCZ_del(NC_GRP_INFO_T *grp) { - int i; + size_t i; int retval; assert(grp && grp->format_grp_info); @@ -608,7 +563,7 @@ ncz_find_grp_var_att(int ncid, int varid, const char *name, int attnum, if (att) { my_att = use_name ? (NC_ATT_INFO_T *)ncindexlookup(attlist, my_norm_name) : - (NC_ATT_INFO_T *)ncindexith(attlist, attnum); + (NC_ATT_INFO_T *)ncindexith(attlist, (size_t)attnum); if (!my_att) return NC_ENOTATT; } diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 2548ad54ba..9aa89721ce 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -211,6 +211,20 @@ typedef struct NCZCONTENT{ } NCZCONTENT; #endif +/* The type for the NC_FORMATX_NCZARR Global State Object */ +typedef struct GlobalNCZarr { /* libnczarr dispatcher specific parameters */ + char dimension_separator; + int default_zarrformat; + NClist* pluginpaths; + NClist* codec_defaults; + NClist* default_libs; + /* All possible HDF5 filter plugins */ + /* Consider onverting to linked list or hash table or + equivalent since very sparse */ + struct NCZ_Plugin** loaded_plugins; //[H5Z_FILTER_MAX+1]; + size_t loaded_plugins_max; /* plugin filter id index. 0= 0 ?nsegs: -nsegs); - int presegs = 0; + size_t abssegs = (size_t)(nsegs >= 0 ?nsegs: -nsegs); + size_t presegs = 0; /* Special case */ if(key == NULL || strlen(key) == 0) goto done; @@ -439,7 +439,7 @@ nczm_segment1(const char* path, char** seg1p) delta = (q-p); if((seg1 = (char*)malloc((size_t)delta+1))==NULL) {ret = NC_ENOMEM; goto done;} - memcpy(seg1,p,delta); + memcpy(seg1,p,(size_t)delta); seg1[delta] = '\0'; if(seg1p) {*seg1p = seg1; seg1 = NULL;} diff --git a/libnczarr/zplugins.c b/libnczarr/zplugins.c new file mode 100644 index 0000000000..8d528c3bfc --- /dev/null +++ b/libnczarr/zplugins.c @@ -0,0 +1,874 @@ +/* Copyright 2003-2018, University Corporation for Atmospheric + * Research. See the COPYRIGHT file for copying and redistribution + * conditions. + */ +/** + * @file @internal netcdf-4 functions for the plugin list. + * + * @author Dennis Heimbigner + */ + +#include "config.h" +#include +#include +#include "zincludes.h" +#include "ncpathmgr.h" +#include "ncpoco.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" +#include "zfilter.h" +#include "ncplugins.h" +#include "zplugins.h" + +#if 0 +#define DEBUG +#define DEBUGF +#define DEBUGL +#define DEBUGPL +#endif + +/**************************************************/ +/* Forward */ + +static int NCZ_plugin_path_initialize(void** statep, const NClist* initialpaths); +static int NCZ_plugin_path_finalize(void** statep); +static int NCZ_plugin_path_read(void* state, size_t* ndirsp, char** dirs); +static int NCZ_plugin_path_write(void* state, size_t ndirs, char** const dirs); + +static int NCZ_load_plugin(const char* path, NCZ_Plugin** plugp); +static int NCZ_unload_plugin(NCZ_Plugin* plugin); +static int NCZ_load_plugin_dir(GlobalNCZarr* gz, const char* path); +static int NCZ_plugin_save(GlobalNCZarr* gz, size_t filterid, NCZ_Plugin* p); +static int getentries(const char* path, NClist* contents); +static int loadcodecdefaults(GlobalNCZarr* gz, const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp); + +#if defined NAMEOPT || defined _WIN32 +static int pluginnamecheck(const char* name); +#endif + +/**************************************************/ +/** + * @file + * @internal + * Internal netcdf zarr plugin path functions. + * + * @author Dennis Heimbigner + */ + +/**************************************************/ + +/**************************************************/ +/* The NCZarr Plugin Path Dispatch table and functions */ +NC_PluginPathDispatch NCZ_pluginpathdispatch = { + NC_FORMATX_NCZARR, + NC_PLUGINPATH_DISPATCH_VERSION, + NCZ_plugin_path_initialize, + NCZ_plugin_path_finalize, + NCZ_plugin_path_read, + NCZ_plugin_path_write, +}; + +NC_PluginPathDispatch* NCZ_pluginpathtable = &NCZ_pluginpathdispatch; + +/**************************************************/ +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NCZ_plugin_path_initialize(void** statep, const NClist* initialpaths) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = NULL; + + assert(statep != NULL); + if(*statep != NULL) goto done; /* already initialized */ + + if((gz = (GlobalNCZarr*)calloc(1,sizeof(GlobalNCZarr)))==NULL) {stat = NC_ENOMEM; goto done;} + + gz->pluginpaths = nclistnew(); + gz->default_libs = nclistnew(); + gz->codec_defaults = nclistnew(); + gz->loaded_plugins = (struct NCZ_Plugin**)calloc(H5Z_FILTER_MAX+1,sizeof(struct NCZ_Plugin*)); + if(gz->loaded_plugins == NULL) {stat = NC_ENOMEM; goto done;} + + /* Preload with set of initial paths (typically coming from HDF5_PLUGIN_PATH) */ + if(nclistlength(initialpaths) > 0) { + size_t i; + for(i=0;ipluginpaths,strdup(dir)); + } + } + + *statep = (void*)gz; gz = NULL; +done: + nullfree(gz); + return THROW(stat); +} + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NCZ_plugin_path_finalize(void** statep) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = NULL; + + assert(statep != NULL); + if(*statep == NULL) goto done; /* already finalized */ + gz = (GlobalNCZarr*)(*statep); + + /* Reclaim all loaded filters */ +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: finalize reclaim:\n"); +#endif + { size_t i; + for(i=1;i<=gz->loaded_plugins_max;i++) { + if(gz->loaded_plugins[i]) { + NCZ_unload_plugin(gz->loaded_plugins[i]); + gz->loaded_plugins[i] = NULL; + } + } + } + /* Reclaim the codec defaults */ + if(nclistlength(gz->codec_defaults) > 0) { + size_t i; + for(i=0;icodec_defaults);i++) { + struct CodecAPI* ca = (struct CodecAPI*)nclistget(gz->codec_defaults,i); + nullfree(ca); + } + } + /* Reclaim the defaults library contents; Must occur as last act */ + if(nclistlength(gz->default_libs) > 0) { + size_t i; + for(i=0;idefault_libs);i++) { + NCPSharedLib* l = (NCPSharedLib*)nclistget(gz->default_libs,i); +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l); +#endif + if(l != NULL) (void)ncpsharedlibfree(l); + } + } + gz->loaded_plugins_max = 0; + nullfree(gz->loaded_plugins); gz->loaded_plugins = NULL; +assert(gz->default_libs != NULL); + nclistfree(gz->default_libs); gz->default_libs = NULL; + nclistfree(gz->codec_defaults); gz->codec_defaults = NULL; + nclistfreeall(gz->pluginpaths); gz->pluginpaths = NULL; + + *statep = NULL; + +done: + nullfree(gz); + return THROW(stat); +} + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * + * @param state the per-dispatcher state + * @param ndirsp return the number of paths in the path list + * @param dirs copy the sequence of directories in the path list into this; caller must free the copied strings + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or zero + * + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. The first time + * with ndirsp not NULL and dirs set to NULL to get the size of + * the path list. The second time with dirs not NULL to get the + * actual sequence of paths. + * Note also that it is not possible to read a subset of the plugin path. + * If the dirs argument is not NULL, then this code assumes it points + * to enough memory to hold the whole plugin path. +*/ + +static int +NCZ_plugin_path_read(void* state, size_t* ndirsp, char** dirs) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = (GlobalNCZarr*)state; + size_t ndirs = 0; + + ndirs = nclistlength(gz->pluginpaths); + if(ndirsp) *ndirsp = ndirs; + + if(ndirs > 0 && dirs != NULL) { + size_t i; + for(i=0;ipluginpaths,i); + dirs[i] = strdup(dir); + } + } + return THROW(stat); +} + +/** + * Empty the current internal path sequence + * and replace with the sequence of directories + * specified in the arguments. + * If ndirs == 0 the path list will be cleared + * + * @param state the per-dispatcher state + * @param ndirs number of entries in dirs arg + * @param dirs the actual directory paths + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or ndirs > 0 and dirs == NULL + * + * @author Dennis Heimbigner +*/ + +static int +NCZ_plugin_path_write(void* state, size_t ndirs, char** const dirs) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = (GlobalNCZarr*)state; + + if(ndirs > 0 && dirs == NULL) {stat = NC_EINVAL; goto done;} + + /* Clear the current path list */ + nclistfreeall(gz->pluginpaths); + gz->pluginpaths = nclistnew(); + + if(ndirs > 0 && dirs != NULL) { + size_t i; + for(i=0;ipluginpaths,nulldup(dirs[i])); + } + +done: + if(gz->pluginpaths == NULL) + gz->pluginpaths = nclistnew(); + return NCTHROW(stat); +} + +/**************************************************/ +/* Filter<->Plugin interface */ + +int +NCZ_load_all_plugins(void) +{ + int ret = NC_NOERR; + size_t i,j; + struct stat buf; + NClist* dirs = nclistnew(); + char* defaultpluginpath = NULL; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + ZTRACE(6,""); + +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); +#endif + + for(i=0;ipluginpaths);i++) { + const char* dir = (const char*)nclistget(gz->pluginpaths,i); + /* Make sure the root is actually a directory */ + errno = 0; + ret = NCstat(dir, &buf); +#if 1 + ZTRACEMORE(6,"stat: ret=%d, errno=%d st_mode=%d",ret,errno,buf.st_mode); +#endif + if(ret < 0) {errno = 0; ret = NC_NOERR; continue;} /* ignore unreadable directories */ + if(! S_ISDIR(buf.st_mode)) + ret = NC_EINVAL; + if(ret) goto done; + + /* Try to load plugins from this directory */ + if((ret = NCZ_load_plugin_dir(gz,dir))) goto done; + } +#ifdef DEBUGL + { size_t i; + fprintf(stderr,"gz->codec_defaults:"); + for(i=0;icodec_defaults);i++) { + struct CodecAPI* codec = (struct CodecAPI*)nclistget(gz->codec_defaults,i); + fprintf(stderr," %d",codec->codec->hdf5id); + } + fprintf(stderr,"\n"); + } +#endif + if(nclistlength(gz->codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */ + /* Search the defaults */ + for(j=0;jcodec_defaults);j++) { + struct CodecAPI* dfalt = (struct CodecAPI*)nclistget(gz->codec_defaults,j); + if(dfalt->codec != NULL) { + const NCZ_codec_t* codec = dfalt->codec; + size_t hdf5id = codec->hdf5id; + NCZ_Plugin* p = NULL; + if(hdf5id <= 0 || hdf5id > gz->loaded_plugins_max) {ret = NC_EFILTER; goto done;} + p = gz->loaded_plugins[hdf5id]; /* get candidate */ + if(p != NULL && p->hdf5.filter != NULL + && p->codec.codec == NULL) { +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path); +#endif + p->codec.codec = codec; + p->codec.codeclib = dfalt->codeclib; + p->codec.defaulted = 1; + } + } + } + } + + /* Mark all plugins for which we do not have both HDF5 and codec */ + { + size_t i; + NCZ_Plugin* p; + for(i=1;iloaded_plugins_max;i++) { + if((p = gz->loaded_plugins[i]) != NULL) { + if(p->hdf5.filter == NULL || p->codec.codec == NULL) { + /* mark this entry as incomplete */ + p->incomplete = 1; +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i, + (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); +#endif + } +#ifdef DEBUGL + else + fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i); +#endif + } + } + } + /* Iniitalize all remaining plugins */ + { + size_t i; + NCZ_Plugin* p; + for(i=1;iloaded_plugins_max;i++) { + if((p = gz->loaded_plugins[i]) != NULL) { + if(p->incomplete) continue; + if(p->hdf5.filter != NULL && p->codec.codec != NULL) { + if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) + p->codec.codec->NCZ_codec_initialize(); +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); +#endif + } + } + } + } + +done: + nullfree(defaultpluginpath); + nclistfreeall(dirs); + errno = 0; + return ZUNTRACE(ret); +} + +/* Load all the filters within a specified directory */ +static int +NCZ_load_plugin_dir(GlobalNCZarr* gz, const char* path) +{ + size_t i; + int stat = NC_NOERR; + size_t pathlen; + NClist* contents = nclistnew(); + char* file = NULL; + + ZTRACE(7,"path=%s",path); + +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin_dir: path=%s\n",path); +#endif + + if(path == NULL) {stat = NC_EINVAL; goto done;} + pathlen = strlen(path); + if(pathlen == 0) {stat = NC_EINVAL; goto done;} + + if((stat = getentries(path,contents))) goto done; + for(i=0;i 0); + nullfree(file); file = NULL; + if((file = (char*)malloc(flen))==NULL) {stat = NC_ENOMEM; goto done;} + file[0] = '\0'; + strlcat(file,path,flen); + strlcat(file,"/",flen); + strlcat(file,name,flen); + /* See if can load the file */ + stat = NCZ_load_plugin(file,&plugin); + switch (stat) { + case NC_NOERR: break; + case NC_ENOFILTER: case NC_ENOTFOUND: + stat = NC_NOERR; + break; /* will cause it to be ignored */ + default: goto done; + } + if(plugin != NULL) { + id = (size_t)plugin->hdf5.filter->id; + if(gz->loaded_plugins[id] == NULL) { + gz->loaded_plugins[id] = plugin; + if(id > gz->loaded_plugins_max) gz->loaded_plugins_max = id; +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin)); +#endif + } else { +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin duplicate: %s\n",printplugin(plugin)); +#endif + NCZ_unload_plugin(plugin); /* its a duplicate */ + } + } else + stat = NC_NOERR; /*ignore failure */ + } + +done: + nullfree(file); + nclistfreeall(contents); + return ZUNTRACE(stat); +} + +int +NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) +{ + int stat = NC_NOERR; + NCZ_Plugin* plugin = NULL; + const H5Z_class2_t* h5class = NULL; + H5PL_type_t h5type = 0; + const NCZ_codec_t** cp = NULL; + const NCZ_codec_t* codec = NULL; + NCPSharedLib* lib = NULL; + int flags = NCP_GLOBAL; + size_t h5id = 0; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + assert(path != NULL && strlen(path) > 0 && plugp != NULL); + + ZTRACE(8,"path=%s",path); + + if(plugp) *plugp = NULL; + +#if defined NAMEOPT || defined _WIN32 + /*triage because visual studio does a popup if the file will not load*/ + if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;} +#endif + + /* load the shared library */ + if((stat = ncpsharedlibnew(&lib))) goto done; + if((stat = ncpload(lib,path,flags))) goto done; + +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib); +#endif + + /* See what we have */ + { + const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); + const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info"); + const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); + const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); + + if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) + {stat = THROW(NC_ENOFILTER); goto done;} + + /* We can have cpd or we can have (gpt && gpi && npi) but not both sets */ + if(cpd != NULL) { + cp = (const NCZ_codec_t**)cpd(); + } else {/* cpd => !gpt && !gpi && !npi */ + if(gpt != NULL && gpi != NULL) { /* get HDF5 info */ + h5type = gpt(); + h5class = gpi(); + /* Verify */ + if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} + if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} + } + if(npi != NULL) {/* get Codec info */ + codec = npi(); + /* Verify */ + if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} + if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} + } + } + } + +#ifdef DEBUGL +fprintf(stderr,">>> DEBUGL: load: %s:",path); +if(h5class) fprintf(stderr,">>> %u",(unsigned)h5class->id); +if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid); +fprintf(stderr,">>> \n"); +#endif + + /* Handle defaults separately */ + if(cp != NULL) { + int used = 0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); +#endif + if((stat = loadcodecdefaults(gz,path,cp,lib,&used))) goto done; + if(used) lib = NULL; + goto done; + } + + if(h5class != NULL && codec != NULL) { + /* Verify consistency of the HDF5 and the Codec */ + if(((size_t)h5class->id) != codec->hdf5id) goto done; /* ignore */ + } + + /* There are several cases to consider: + 1. This library has both HDF5 API and Codec API => merge + 2. This library has HDF5 API only and Codec API was already found in another library => merge + 3. This library has Codec API only and HDF5 API was already found in another library => merge + */ + + /* Get any previous plugin entry for this id; may be NULL */ + if(h5class != NULL) { + h5id = (size_t)h5class->id; + if((stat = NCZ_plugin_loaded(h5id,&plugin))) goto done; + } else if(codec != NULL) { + h5id = (size_t)codec->hdf5id; + if((stat = NCZ_plugin_loaded(h5id,&plugin))) goto done; + } + + if(plugin == NULL) { + /* create new entry */ + if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;} + } + + /* Fill in the plugin */ + if(h5class != NULL && plugin->hdf5.filter == NULL) { + plugin->hdf5.filter = h5class; + plugin->hdf5.hdf5lib = lib; + lib = NULL; + } + if(codec != NULL && plugin->codec.codec == NULL) { + plugin->codec.codec = codec; + plugin->codec.codeclib = lib; + lib = NULL; + } +#ifdef DEBUGL + if(plugin) + fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin)); +#endif + /* Cleanup */ + if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */ + plugin->codec.codeclib = NULL; + if((stat=NCZ_plugin_save(gz,h5id,plugin))) goto done; + plugin = NULL; + +done: + if(lib) + (void)ncpsharedlibfree(lib); + if(plugin) NCZ_unload_plugin(plugin); + return ZUNTRACEX(stat,"plug=%p",*plugp); +} + +int +NCZ_unload_plugin(NCZ_Plugin* plugin) +{ + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + ZTRACE(9,"plugin=%p",plugin); + + if(plugin) { +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin)); +#endif + if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize) + plugin->codec.codec->NCZ_codec_finalize(); + if(plugin->hdf5.filter != NULL) gz->loaded_plugins[plugin->hdf5.filter->id] = NULL; + if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib); + if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); +memset(plugin,0,sizeof(NCZ_Plugin)); + free(plugin); + } + return ZUNTRACE(NC_NOERR); +} + +int +NCZ_plugin_loaded(size_t filterid, NCZ_Plugin** pp) +{ + int stat = NC_NOERR; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + struct NCZ_Plugin* plug = NULL; + ZTRACE(6,"filterid=%d",filterid); + if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) + {stat = NC_EINVAL; goto done;} + if(filterid <= gz->loaded_plugins_max) + plug = gz->loaded_plugins[filterid]; + if(pp) *pp = plug; +done: + return ZUNTRACEX(stat,"plugin=%p",*pp); +} + +int +NCZ_plugin_loaded_byname(const char* name, NCZ_Plugin** pp) +{ + int stat = NC_NOERR; + size_t i; + struct NCZ_Plugin* plug = NULL; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + ZTRACE(6,"pluginname=%s",name); + if(name == NULL) {stat = NC_EINVAL; goto done;} + for(i=1;i<=gz->loaded_plugins_max;i++) { + if (!gz->loaded_plugins[i]) continue; + if(!gz->loaded_plugins[i] || !gz->loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */ + if(strcmp(name, gz->loaded_plugins[i]->codec.codec->codecid) == 0) + {plug = gz->loaded_plugins[i]; break;} + } + if(pp) *pp = plug; +done: + return ZUNTRACEX(stat,"plugin=%p",*pp); +} + +static int +NCZ_plugin_save(GlobalNCZarr* gz, size_t filterid, NCZ_Plugin* p) +{ + int stat = NC_NOERR; + + ZTRACE(6,"filterid=%d p=%p",filterid,p); + if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) + {stat = NC_EINVAL; goto done;} + if(filterid > gz->loaded_plugins_max) gz->loaded_plugins_max = filterid; + gz->loaded_plugins[filterid] = p; +done: + return ZUNTRACE(stat); +} + +static int +loadcodecdefaults(GlobalNCZarr* gz, const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp) +{ + int stat = NC_NOERR; + int lib_used = 0; + + nclistpush(gz->default_libs,lib); + for(;*cp;cp++) { + struct CodecAPI* c0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); +#endif + c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); + if(c0 == NULL) {stat = NC_ENOMEM; goto done;} + c0->codec = *cp; + c0->codeclib = lib; + lib_used = 1; /* remember */ + nclistpush(gz->codec_defaults,c0); c0 = NULL; + } +done: + if(lib_usedp) *lib_usedp = lib_used; + return THROW(stat); +} + +#if defined NAMEOPT || defined _WIN32 +static int +pluginnamecheck(const char* name) +{ + size_t count,len; + long i; + const char* p; + if(name == NULL) return 0; + /* get basename */ + p = strrchr(name,'/'); + if(p != NULL) name = (p+1); + len = strlen(name); + if(len == 0) return 0; + i = (long)(len-1); + count = 1; + p = name+i; + for(;i>=0;i--,count++,p--) { + char c = *p; + if(c == '/') break; + if(c == '.') { + if(count >= 3 && memcmp(p,".so",3)==0) return 1; + if(count >= 4 && memcmp(p,".dll",4)==0) return 1; + if(count >= 6 && memcmp(p,".dylib",6)==0) return 1; + } + } + return 0; +} +#endif + +/**************************************************/ +/* +Get entries in a path that is assumed to be a directory. +*/ + +#ifdef _WIN32 + +static int +getentries(const char* path, NClist* contents) +{ + /* Iterate over the entries in the directory */ + int ret = NC_NOERR; + errno = 0; + WIN32_FIND_DATA FindFileData; + HANDLE dir = NULL; + char* ffpath = NULL; + char* lpath = NULL; + size_t len; + char* d = NULL; + + ZTRACE(6,"path=%s",path); + + /* We need to process the path to make it work with FindFirstFile */ + len = strlen(path); + /* Need to terminate path with '/''*' */ + ffpath = (char*)malloc(len+2+1); + memcpy(ffpath,path,len); + if(path[len-1] != '/') { + ffpath[len] = '/'; + len++; + } + ffpath[len] = '*'; len++; + ffpath[len] = '\0'; + + /* localize it */ + if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done; + dir = FindFirstFile(lpath, &FindFileData); + if(dir == INVALID_HANDLE_VALUE) { + /* Distinquish not-a-directory from no-matching-file */ + switch (GetLastError()) { + case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */ + ret = NC_NOERR; + goto done; + case ERROR_DIRECTORY: /* not a directory */ + default: + ret = NC_EEMPTY; + goto done; + } + } + do { + char* p = NULL; + const char* name = NULL; + name = FindFileData.cFileName; + if(strcmp(name,".")==0 || strcmp(name,"..")==0) + continue; + nclistpush(contents,strdup(name)); + } while(FindNextFile(dir, &FindFileData)); + +done: + if(dir) FindClose(dir); + nullfree(lpath); + nullfree(ffpath); + nullfree(d); + errno = 0; + return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); +} + +#else /* !_WIN32 */ + +int +getentries(const char* path, NClist* contents) +{ + int ret = NC_NOERR; + errno = 0; + DIR* dir = NULL; + + ZTRACE(6,"path=%s",path); + + dir = NCopendir(path); + if(dir == NULL) + {ret = (errno); goto done;} + for(;;) { + const char* name = NULL; + struct dirent* de = NULL; + errno = 0; + de = readdir(dir); + if(de == NULL) + {ret = (errno); goto done;} + if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0) + continue; + name = de->d_name; + nclistpush(contents,strdup(name)); + } +done: + if(dir) NCclosedir(dir); + errno = 0; + return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); +} +#endif /*_WIN32*/ + +/**************************************************/ + + +#if defined(DEBUGF) || defined(DEBUGL) + +const char* +printplugin(const NCZ_Plugin* plugin) +{ + static char plbuf[4096]; + char plbuf2[2000]; + char plbuf1[2000]; + + if(plugin == NULL) return "plugin=NULL"; + plbuf2[0] = '\0'; plbuf1[0] = '\0'; + if(plugin->hdf5.filter) + snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name); + if(plugin->codec.codec) + snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id); + snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2); + return plbuf; +} + +static char* +printparams(size_t nparams, const unsigned* params) +{ + static char ppbuf[4096]; + if(nparams == 0) + snprintf(ppbuf,4096,"{0,%p}",params); + else + snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params)); + return ppbuf; +} + +static char* +printnczparams(const NCZ_Params p) +{ + return printparams(p.nparams,p.params); +} + +static const char* +printcodec(const NCZ_Codec c) +{ + static char pcbuf[4096]; + snprintf(pcbuf,sizeof(pcbuf),"{id=%s codec=%s}", + c.id,NULLIFY(c.codec)); + return pcbuf; +} + +static const char* +printhdf5(const NCZ_HDF5 h) +{ + static char phbuf[4096]; + snprintf(phbuf,sizeof(phbuf),"{id=%u visible=%s working=%s}", + h.id, printnczparams(h.visible), printnczparams(h.working)); + return phbuf; +} +#endif /* defined(DEBUGF) || defined(DEBUGL) */ + +/* Suppress selected unused static functions */ +static void static_unused(void) +{ + void* p = NULL; + p = p; + p = static_unused; +#if defined(DEBUGF) || defined(DEBUGL) +(void)printplugin(NULL); +(void)printparams(0, NULL); +(void)printnczparams(const NCZ_Params p); +(void)printcodec(const NCZ_Codec c); +(void)printhdf5(const NCZ_HDF5 h); +#endif +} + diff --git a/libnczarr/zplugins.h b/libnczarr/zplugins.h new file mode 100644 index 0000000000..3ad26abd77 --- /dev/null +++ b/libnczarr/zplugins.h @@ -0,0 +1,45 @@ +/* Copyright 2018-2018 University Corporation for Atmospheric + Research/Unidata. */ + +/** + * @file This header file containsplugin related macros, types, and prototypes for + * the plugin code in libnczarr. This header should not be included in + * code outside libnczarr. + * + * @author Dennis Heimbigner + */ + +#ifndef ZPLUGIN_H +#define ZPLUGIN_H + + +/* zplugin.c */ + +/* Pluginlist management */ + +/* Opaque Handles */ +struct H5Z_class2_t; +struct NCZ_codec_t; +struct NCPSharedLib; + +/* Hold the loaded filter plugin information */ +typedef struct NCZ_Plugin { + int incomplete; + struct HDF5API { + const struct H5Z_class2_t* filter; + struct NCPSharedLib* hdf5lib; /* source of the filter */ + } hdf5; + struct CodecAPI { + int defaulted; /* codeclib was a defaulting library */ + int ishdf5raw; /* The codec is the hdf5raw codec */ + const struct NCZ_codec_t* codec; + struct NCPSharedLib* codeclib; /* of the codec; null if same as hdf5 */ + } codec; +} NCZ_Plugin; + +int NCZ_load_all_plugins(void); +int NCZ_plugin_loaded(size_t filterid, NCZ_Plugin** pp); +int NCZ_plugin_loaded_byname(const char* name, NCZ_Plugin** pp); + +#endif /*ZPLUGIN_H*/ + diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index c9d55ee751..a79be7f69c 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1533,7 +1533,7 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) /* Capture dimension_separator (must precede chunk cache creation) */ { NCglobalstate* ngs = NC_getglobalstate(); - assert(ngs != NULL); + GlobalNCZarr* ncz = (GlobalNCZarr*)ngs->formatxstate.state[NC_FORMATX_NCZARR]; zvar->dimension_separator = 0; if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))<0) {stat = NC_EINVAL; goto done;} if(jvalue != NULL) { @@ -1543,7 +1543,7 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) } /* If value is invalid, then use global default */ if(!islegaldimsep(zvar->dimension_separator)) - zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */ + zvar->dimension_separator = ncz->dimension_separator; /* use global value */ assert(islegaldimsep(zvar->dimension_separator)); /* we are hosed */ } @@ -1650,7 +1650,6 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) if(var->filters == NULL) var->filters = (void*)nclistnew(); if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); chainindex = 0; /* track location of filter in the chain */ - if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"filters",&jvalue))<0) {stat = NC_EINVAL; goto done;} if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { int k; @@ -1672,7 +1671,6 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) #ifdef NETCDF_ENABLE_NCZARR_FILTERS { if(var->filters == NULL) var->filters = (void*)nclistnew(); - if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"compressor",&jfilter))<0) {stat = NC_EINVAL; goto done;} if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index eb26eb7eda..dac90daf3e 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -294,7 +294,8 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, char norm_name[NC_MAX_NAME + 1]; int d; int retval; - NCglobalstate* gstate = NC_getglobalstate(); + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; ZTRACE(1,"ncid=%d name=%s xtype=%d ndims=%d dimids=%s",ncid,name,xtype,ndims,nczprint_idvector(ndims,dimidsp)); @@ -381,7 +382,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, zvar->common.file = h5; zvar->scalar = (ndims == 0 ? 1 : 0); - zvar->dimension_separator = gstate->zarr.dimension_separator; + zvar->dimension_separator = gz->dimension_separator; assert(zvar->dimension_separator != 0); /* Set these state flags for the var. */ @@ -457,7 +458,7 @@ var->type_info->rc++; zvar->chunksize = zvar->chunkproduct * var->type_info->size; /* Set cache defaults */ - var->chunkcache = gstate->chunkcache; + var->chunkcache = gs->chunkcache; /* Create the cache */ if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache))) diff --git a/libsrc4/nc4dispatch.c b/libsrc4/nc4dispatch.c index 2aa5987824..3407960b32 100644 --- a/libsrc4/nc4dispatch.c +++ b/libsrc4/nc4dispatch.c @@ -25,7 +25,6 @@ extern NC_Dispatch UDF0_DISPATCH; extern NC_Dispatch UDF1_DISPATCH; #endif /* USE_UDF1 */ - /** * @internal Initialize netCDF-4. If user-defined format(s) have been * specified in configure, load their dispatch table(s). @@ -62,6 +61,7 @@ NC4_initialize(void) } #endif #endif + NC_initialize_reserved(); return ret; } diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index caa75aff18..f30194385d 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -40,6 +40,8 @@ IF(NETCDF_BUILD_UTILITIES) ADD_SH_TEST(nc_test4 tst_misc) build_bin_test(tst_fillonly) ADD_SH_TEST(nc_test4 test_fillonly) +# H5 and nczarr Fixed string support + build_bin_test(build_fixedstring) ADD_SH_TEST(nc_test4 tst_fixedstring) IF(USE_HDF5 AND NETCDF_ENABLE_FILTER_TESTING) build_bin_test(tst_filterparser) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 683b9a1145..a07e20856d 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -68,7 +68,8 @@ TESTS += run_grp_rename.sh tst_misc.sh check_PROGRAMS += tst_fillonly TESTS += test_fillonly.sh -# H5 and nczarr Fixed string support +# H5 Fixed string support +check_PROGRAMS += build_fixedstring TESTS += tst_fixedstring.sh # Szip Tests (requires ncdump) @@ -143,7 +144,7 @@ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \ ref_filter_order_create.txt ref_filter_order_read.txt ref_any.cdl \ tst_specific_filters.sh tst_unknown.sh tst_virtual_datasets.c \ noop1.cdl unknown.cdl tst_broken_files.c ref_bloscx.cdl \ -tst_bloscfail.sh tst_fixedstring.sh ref_fixedstring.h5 \ +tst_bloscfail.sh tst_fixedstring.sh \ ref_fixedstring.cdl tst_filterinstall.sh tst_filter_vlen.sh \ tst_filter_misc.sh run_zstd_test.sh run_par_warn_test.sh.in \ ref_tmp_tst_warn_out.txt @@ -159,7 +160,7 @@ floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \ testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \ testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \ tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp \ -tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl +tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl ref_fixedstring.h5 DISTCLEANFILES = findplugin.sh run_par_test.sh run_par_warn_test.sh @@ -168,13 +169,3 @@ clean-local: # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ - -# The (otherwise unused) program build_fixedstring.c -# is used to generate the test file ref_fixedstring.h5. -# That test file is build and included as part of the distribution, -# so the build_fixedstring.c program generally does not need to -# be executed unless the test file needs to be modified.. - -check_PROGRAMS += build_fixedstring -ref_fixedstring.h5: build_fixedstring.c - ${srcdir}/buildfixedstring diff --git a/nc_test4/ref_fixedstring.cdl b/nc_test4/ref_fixedstring.cdl index 09f6fdc34a..8912999bbc 100644 --- a/nc_test4/ref_fixedstring.cdl +++ b/nc_test4/ref_fixedstring.cdl @@ -1,9 +1,16 @@ netcdf ref_fixedstring { dimensions: - phony_dim_0 = 3 ; + phony_dim_0 = 4 ; variables: - string test(phony_dim_0) ; + string v1 ; + string vn(phony_dim_0) ; + +// global attributes: + :att1 = "abcd" ; + string :attn = "abcd", "efgh", "ijkl", "mnop" ; data: - test = "foo", "bar", "baz" ; + v1 = "abcd" ; + + vn = "abcd", "efgh", "ijkl", "mnop" ; } diff --git a/nc_test4/ref_fixedstring.h5 b/nc_test4/ref_fixedstring.h5 deleted file mode 100644 index b2cc269362..0000000000 Binary files a/nc_test4/ref_fixedstring.h5 and /dev/null differ diff --git a/nc_test4/tst_fixedstring.sh b/nc_test4/tst_fixedstring.sh index f27e456e9e..038299a191 100755 --- a/nc_test4/tst_fixedstring.sh +++ b/nc_test4/tst_fixedstring.sh @@ -6,10 +6,11 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e # Note, the test file for this is ref_fixedstring.h5 -# But is is generated by the (otherwise unused) program -# ../h5_test/tst_h_fixedstrings.c. +# and is generated on the fly by build_fixedstring. +${execdir}/build_fixedstring echo "*** Test reading a file with HDF5 fixed length strings" rm -f ./tmp_fixedstring.cdl -$NCDUMP ${srcdir}/ref_fixedstring.h5 > ./tmp_fixedstring.cdl +$NCDUMP ${execdir}/ref_fixedstring.h5 > ./tmp_fixedstring.cdl diff -b -w ${srcdir}/ref_fixedstring.cdl ./tmp_fixedstring.cdl +rm -f ${execdir}/ref_fixedstring.h5 diff --git a/unit_test/CMakeLists.txt b/unit_test/CMakeLists.txt index 39074424d8..9bda2f1255 100644 --- a/unit_test/CMakeLists.txt +++ b/unit_test/CMakeLists.txt @@ -41,11 +41,19 @@ IF(NETCDF_BUILD_UTILITIES) build_bin_test(aws_config) add_sh_test(unit_test run_aws_config) ENDIF() + + # plugin path test + if(NETCDF_ENABLE_NCZARR AND NETCDF_ENABLE_HDF5) + build_bin_test(tst_pluginpaths ${XGETOPTSRC}) + add_sh_test(unit_test run_pluginpaths) + endif() ENDIF() # Performance tests +if(BUILD_BENCHMARKS) add_bin_test(unit_test tst_exhash timer_utils.c) add_bin_test(unit_test tst_xcache timer_utils.c) +endif() FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) diff --git a/unit_test/Makefile.am b/unit_test/Makefile.am index 6061edaf41..4752397fd3 100644 --- a/unit_test/Makefile.am +++ b/unit_test/Makefile.am @@ -6,7 +6,7 @@ # functions, to allow Windows to work. Since we have not extern'd # these functions, they will only be run under the autotools build. -# Ed Hartnett 8/9/19 +# Dennis Heimbigner 8/27/2024 #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose @@ -25,12 +25,15 @@ TESTS = check_PROGRAMS += tst_nclist test_ncuri test_pathcvt +TESTS += tst_nclist test_ncuri test_pathcvt + # Performance tests +if BUILD_BENCHMARKS check_PROGRAMS += tst_exhash tst_xcache tst_exhash_SOURCES = tst_exhash.c timer_utils.c timer_utils.h tst_xcache_SOURCES = tst_xcache.c timer_utils.c timer_utils.h - -TESTS += tst_nclist test_ncuri test_pathcvt tst_exhash tst_xcache +TESTS += tst_exhash tst_xcache +endif if USE_HDF5 check_PROGRAMS += tst_nc4internal tst_reclaim @@ -38,6 +41,14 @@ TESTS += tst_nc4internal TESTS += run_reclaim_tests.sh endif # USE_HDF5 +# Test for HDF5 OR NCZARR enabled +if USE_HDF5 +if NETCDF_ENABLE_NCZARR +check_PROGRAMS += tst_pluginpaths +TESTS += run_pluginpaths.sh +endif +endif + if NETCDF_ENABLE_S3 if NETCDF_ENABLE_S3_TESTALL check_PROGRAMS += test_s3sdk @@ -47,10 +58,11 @@ check_PROGRAMS += aws_config TESTS += run_aws_config.sh endif -EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh +EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh run_aws_config.sh run_pluginpaths.sh EXTRA_DIST += nctest_netcdf4_classic.nc reclaim_tests.cdl +EXTRA_DIST += ref_read.txt ref_write.txt -CLEANFILES = reclaim_tests*.txt reclaim_tests.nc +CLEANFILES = reclaim_tests*.txt reclaim_tests.nc tmp_*.txt # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/unit_test/aws_config.c b/unit_test/aws_config.c index 1eb6c35c69..c9764ee18b 100644 --- a/unit_test/aws_config.c +++ b/unit_test/aws_config.c @@ -46,8 +46,7 @@ main(int argc, char** argv) int c = 0,stat = NC_NOERR; /* Load RC and .aws/config */ - CHECK(nc_initialize()); - CHECK(NC_s3sdkinitialize()); + CHECK(nc_initialize()); /* Will invoke NC_s3sdkinitialize()); */ NCglobalstate* gs = NC_getglobalstate(); //Not checking, aborts if ~/.aws/config doesn't exist CHECK(NC_aws_load_credentials(gs)); diff --git a/unit_test/ref_read.txt b/unit_test/ref_read.txt new file mode 100644 index 0000000000..0d08a18833 --- /dev/null +++ b/unit_test/ref_read.txt @@ -0,0 +1,5 @@ +testread(hdf5): /zero:/one:/two:/hdf5:/three:/four +testread(nczarr): /zero:/one:/two:/nczarr:three:/four:/five +testread(all): +testread(hdf5): /zero:/one:/two:/hdf5:/three:/four +testread(nczarr): /zero:/one:/two:/nczarr:three:/four:/five diff --git a/unit_test/ref_write.txt b/unit_test/ref_write.txt new file mode 100644 index 0000000000..b2d303dc0e --- /dev/null +++ b/unit_test/ref_write.txt @@ -0,0 +1,9 @@ +testwrite(hdf5): before: /zero:/one:/two:/hdf5:/three:/four +testwrite(hdf5): after: /zero:/one:/two:/hdf5:/three:/four:/modhdf5 +testwrite(nczarr): before: /zero:/one:/two:/nczarr:three:/four:/five +testwrite(nczarr): after: /modnczarr:/zero:/one:/two:/nczarr:three:/four:/five +testwrite(all): +testwrite(hdf5): before: /zero:/one:/two:/hdf5:/three:/four +testwrite(hdf5): after: /zero:/one:/two:/hdf5:/three:/four:/modhdf5 +testwrite(nczarr): before: /zero:/one:/two:/nczarr:three:/four:/five +testwrite(nczarr): after: /modnczarr:/zero:/one:/two:/nczarr:three:/four:/five diff --git a/unit_test/run_aws_config.sh b/unit_test/run_aws_config.sh index 44b3526643..94060d3011 100755 --- a/unit_test/run_aws_config.sh +++ b/unit_test/run_aws_config.sh @@ -7,7 +7,7 @@ set -e #CMD="valgrind --leak-check=full" -isolate "testdir_ut_aws_config" +isolate "testdir_aws_config" THISDIR=`pwd` cd $ISOPATH @@ -19,6 +19,10 @@ test_cleanup() { } trap test_cleanup EXIT +test_cleanup + +mkdir -p $THISDIR/.aws + cat << 'EOF' > $THISDIR/.aws/config [uni] region = somewhere-1 @@ -37,31 +41,37 @@ endpoint_url = https://endpoint.example.com/ EOF cat << 'EOF' > $THISDIR/.aws/credentials -[play] -aws_access_key_id = DummyKeys -aws_secret_access_key = DummySecret - [uni] region = somewhere-2 endpoint_url = https://example.com/bucket/prefix/2 key = value-overwritten + +[play] +aws_access_key_id = DummyKeys +aws_secret_access_key = DummySecret EOF echo -e "Testing loading AWS configuration in ${THISDIR}/.aws/config" -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=unidata ${CMD} ${execdir}/aws_config endpoint_url region dummy_key +export NC_TEST_AWS_DIR=${THISDIR} +export AWS_PROFILE=unidata +${CMD} ${execdir}/aws_config endpoint_url region dummy_key echo "Status: $?" - -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=play ${CMD} ${execdir}/aws_config endpoint_url region +exit +export AWS_PROFILE=play +${CMD} ${execdir}/aws_config endpoint_url region echo "Status: $?" -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config endpoint_url region key +export AWS_PROFILE=uni +${CMD} ${execdir}/aws_config endpoint_url region key echo "Status: $?" -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated +export AWS_PROFILE=uni +${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated echo "Status: $?" # Will use profile=no -NC_TEST_AWS_DIR=${THISDIR} ${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no' +unset AWS_PROFILE +${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no' echo "Status: $?" echo -e "Finished" diff --git a/unit_test/run_pluginpaths.sh b/unit_test/run_pluginpaths.sh new file mode 100755 index 0000000000..aded28700d --- /dev/null +++ b/unit_test/run_pluginpaths.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# Test the programmatic API for manipulating the plugin paths. +# WARNING: This file is also used to build nczarr_test/run_pluginpath.sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +IMPLS= +if test "x$FEATURE_HDF5" = xyes ; then IMPLS="$IMPLS hdf5"; fi +if test "x$FEATURE_NCZARR" = xyes ; then IMPLS="$IMPLS nczarr"; fi +# Remove leading blank +IMPLS=`echo "$IMPLS" | cut -d' ' -f2,3` +echo "IMPLS=|$IMPLS|" + +#VERBOSE=1 + +DFALT="/zero;/one;/two;/three;/four" +DFALTHDF5="/zero;/one;/two;/hdf5;/three;/four" +DFALTNCZARR="/zero;/one;/two;/nczarr;three;/four;/five" + +if test "x$TESTNCZARR" = x1 ; then +. "$srcdir/test_nczarr.sh" +s3isolate "testdir_pluginpath" +THISDIR=`pwd` +cd $ISOPATH +fi + +TP="${execdir}/tst_pluginpaths" + +filenamefor() { + # tmp|ref_action + eval "filename=${1}_$2" +} + +dfaltfor() { + case "$1" in + hdf5) eval "dfalt=\"$DFALTHDF5\"" ;; + nczarr) eval "dfalt=\"$DFALTNCZARR\"" ;; + all) eval "dfalt=\"$DFALT\"" ;; + esac +} + +modfor() { + local formatx="$1" + local dfalt="$2" + case "$formatx" in + hdf5) mod="${dfalt};/modhdf5" ;; + nczarr) mod="/modnczarr;${dfalt}" ;; + all) mode="${dfalt}" ;; + esac +} + +##### + +testread() { + local formatx="$1" + filenamefor tmp read + dfaltfor $formatx + if test "$formatx" = all ; then + echo "testread(${formatx}): " >> ${filename}.txt + for f in $IMPLS ; do testread $f ; done + elif test "x$FEATURE_HDF5" && test "$formatx" = hdf5 ; then + echon "testread(${formatx}): " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt ; + elif test "x$FEATURE_NCZARR" && test "$formatx" = nczarr ; then + echon "testread(${formatx}): " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt ; + fi +} + +testwrite() { + local formatx="$1" + filenamefor tmp write + dfaltfor $formatx + modfor $formatx "$dfalt" + if test "$formatx" = all ; then + echo "testwrite(${formatx}): " >> ${filename}.txt + for f in $IMPLS ; do testwrite $f ; done + elif test "x$FEATURE_HDF5" && test "$formatx" = hdf5 ; then + echon "testwrite(${formatx}): before: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt + echon "testwrite(${formatx}): after: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${mod},read" >> ${filename}.txt + elif test "x$FEATURE_NCZARR" && test "$formatx" = nczarr ; then + echon "testwrite(${formatx}): before: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt + echon "testwrite(${formatx}): after: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${mod},read" >> ${filename}.txt + fi +} + +######################### + +cleanup() { + rm -f tmp_*.txt +} + +init() { + cleanup +} + +# Verify output for a specific action +verify() { + for action in read write ; do + if diff -wBb ${srcdir}/ref_${action}.txt tmp_${action}.txt ; then + echo "***PASS: $action" + else + echo "***FAIL: $action" + exit 1 + fi + done +} + +init +for fx in $IMPLS all ; do + testread $fx + testwrite $fx +done +verify +cleanup diff --git a/unit_test/tst_pluginpaths.c b/unit_test/tst_pluginpaths.c new file mode 100644 index 0000000000..0b508d9cbc --- /dev/null +++ b/unit_test/tst_pluginpaths.c @@ -0,0 +1,331 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_aux.h" + +#undef DEBUG + +/* Define max number of -x actions */ +#define NACTIONS 64 +/* Define max length of -x action string */ +#define NACTIONLEN 4096 + +/* Define max no. of dirs in path list */ +#define NDIRSMAX 64 + +typedef enum Action { +ACT_NONE=0, +ACT_READ=1, +ACT_WRITE=2, +/* Synthetic Actions */ +ACT_CLEAR=3, +ACT_FORMATX=4, +} Action; + +static struct ActionTable { + Action op; + const char* opname; +} actiontable[] = { +{ACT_NONE,"none"}, +{ACT_READ,"read"}, +{ACT_WRITE,"write"}, +{ACT_CLEAR,"clear"}, +{ACT_FORMATX,"formatx"}, +{ACT_NONE,NULL} +}; + +static struct FormatXTable { + const char* name; + int formatx; +} formatxtable[] = { +{"all",0}, +{"hdf5",NC_FORMATX_NC_HDF5}, +{"nczarr",NC_FORMATX_NCZARR}, +{"zarr",NC_FORMATX_NCZARR}, +{NULL,0} +}; + +/* command line options */ +struct Dumpptions { + int debug; + size_t nactions; + int formatx; /* default formatx */ + struct Execute { + Action action; + char* name; + char* arg; + } actions[NACTIONS]; +} dumpoptions; + +/* Forward */ + +#define NCCHECK(expr) nccheck((expr),__LINE__) +static void ncbreakpoint(int stat) {stat=stat;} +static int nccheck(int stat, int line) +{ + if(stat) { + fprintf(stderr,"%d: %s\n",line,nc_strerror(stat)); + fflush(stderr); + ncbreakpoint(stat); + exit(1); + } + return stat; +} + +static void +pluginusage(void) +{ + fprintf(stderr,"usage: tst_pluginpath [-d] -x [:],[:]...\n"); + fprintf(stderr,"\twhere is one of: read | write | clear| or formatx.\n"); + fprintf(stderr,"\twhere is arbitrary string (with '\\,' to escape commas); arg can be missing or empty.\n"); + exit(1); +} + +static int +decodeformatx(const char* name) +{ + struct FormatXTable* p = formatxtable; + for(;p->name != NULL;p++) { + if(strcasecmp(p->name,name)==0) return p->formatx; + } + return -1; +} + +static Action +decodeop(const char* name) +{ + struct ActionTable* p = actiontable; + for(;p->opname != NULL;p++) { + if(strcasecmp(p->opname,name)==0) return p->op; + } + return ACT_NONE; +} + +/* Unescape all escaped characters in s */ +static void +descape(char* s) +{ + char* p; + char* q; + if(s == NULL) goto done; + for(p=s,q=s;*p;) { + if(*p == '\\' && p[1] != '\0') p++; + *q++ = *p++; + } + *q = *p; /* nul terminate */ +done: + return; +} + +/* A version of strchr that takes escapes into account */ +static char* +xstrchr(char* s, char c) +{ + int leave; + char* p; + for(leave=0,p=s;!leave;p++) { + switch (*p) { + case '\\': p++; break; + case '\0': return NULL; break; + default: if(*p == c) return p; break; + } + } + return NULL; +} + +static void +parseactionlist(const char* cmds0) +{ + size_t i,cmdlen; + char cmds[NACTIONLEN+2]; + char* p; + char* q; + size_t ncmds; + int leave; + + memset(cmds,0,sizeof(cmds)); + if(cmds0 == NULL) cmdlen = 0; else cmdlen = strlen(cmds0); + if(cmdlen == 0) {fprintf(stderr,"error: -x must have non-empty argument.\n"); pluginusage();} + if(cmdlen > NACTIONLEN) {fprintf(stderr,"error: -x argument too lone; max is %zu\n",(size_t)NACTIONLEN); pluginusage();} + strncpy(cmds,cmds0,cmdlen); + /* split into command + formatx + arg strings and count */ + ncmds = 0; + for(leave=0,p=cmds;!leave;p=q) { + q = xstrchr(p,','); + if(q == NULL) { + q = cmds+cmdlen; /* point to trailing nul */ + leave = 1; + } else { + *q++ = '\0'; /* overwrite ',' and skip to start of the next command*/ + } + ncmds++; + } + if(ncmds > NACTIONS) {fprintf(stderr,"error: -x must have not more than %zu commands.\n",(size_t)NACTIONS); pluginusage();} + dumpoptions.nactions = ncmds; + /* Now process each command+formatx+arg triple */ + for(p=cmds,i=0;i NACTIONLEN) {fprintf(stderr,"error: -x cmd '%s' too long; max is %zu\n",p,(size_t)NACTIONLEN); pluginusage();} + /* search for ':' taking escapes into account */ + q = xstrchr(p,':'); + if(q == NULL) + q = p+clen; /* point to trailing nul */ + else + *q++ = '\0'; /* overwrite ':' and skip to start of the arg*/ + dumpoptions.actions[i].name = nulldup(p); + /* Get the argument, if any */ + alen = strlen(q); + if(alen > 0) { + dumpoptions.actions[i].arg = strdup(q); + } + p += (clen+1); /* move to next cmd+arg pair */ + } + /* De-escape names and args and compute action enum */ + for(i=0;iarg); + if(dumpoptions.formatx < 0) {fprintf(stderr,"error: formatx: illegal argument: %s\n",action->arg); pluginusage();} + return NCCHECK(stat); +} + +static int +actionread(const struct Execute* action) +{ + int stat = NC_NOERR; + size_t ndirs; + char** dirs = NULL; + char* text = NULL; + size_t textlen; + + if((stat=nc_plugin_path_read(dumpoptions.formatx,&ndirs,NULL))) goto done; + assert(ndirs > 0); + if((dirs = (char**)calloc(ndirs,sizeof(char*)))==NULL) {stat = NC_ENOMEM; goto done;} + if((stat = nc_plugin_path_read(dumpoptions.formatx,&ndirs,dirs))) goto done; + if((stat = ncaux_plugin_path_tostring(ndirs,dirs,0,&textlen,NULL))) goto done; + if((text = (char*)malloc(textlen))==NULL) {stat = NC_ENOMEM; goto done;} + if((stat = ncaux_plugin_path_tostring(ndirs,dirs,0,&textlen,text))) goto done; + if(stat == NC_NOERR) printf("%s\n",text); +done: + nullfree(text); + ncaux_plugin_path_freestringvec(ndirs,dirs); + return NCCHECK(stat); +} + +static int +actionwrite(const struct Execute* action) +{ + int stat = NC_NOERR; + const char* text = action->arg; + size_t ndirs; + char** dirs = NULL; + + if(text == NULL) text = ""; + if((stat=ncaux_plugin_path_parse(text,0,&ndirs,NULL))) goto done; + if(ndirs > 0) { + if((dirs = (char**)calloc(ndirs,sizeof(char*)))==NULL) {stat = NC_ENOMEM; goto done;} + if((stat=ncaux_plugin_path_parse(text,0,&ndirs,dirs))) goto done; + } + if((stat=nc_plugin_path_write(dumpoptions.formatx,ndirs,dirs))) goto done; + +done: + ncaux_plugin_path_freestringvec(ndirs,dirs); + return NCCHECK(stat); +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + int c; + size_t i; + + /* Init options */ + memset((void*)&dumpoptions,0,sizeof(dumpoptions)); + + while ((c = getopt(argc, argv, "dvx:X:")) != EOF) { + switch(c) { + case 'd': + dumpoptions.debug = 1; + break; + case 'v': + pluginusage(); + goto done; + case 'x': + parseactionlist(optarg); + break; + case '?': + fprintf(stderr,"unknown option\n"); + {stat = NC_EINVAL; goto done;} + } + } + + for(i=0;i>>> [%zu] %s(%d) : %s\n",i, + dumpoptions.actions[i].name, + dumpoptions.actions[i].action, + dumpoptions.actions[i].arg); +#endif + switch (dumpoptions.actions[i].action) { + default: + fprintf(stderr,"Illegal action: %s\n",dumpoptions.actions[i].name); + pluginusage(); + break; + case ACT_CLEAR: if((stat=actionclear(&dumpoptions.actions[i]))) goto done; break; + case ACT_FORMATX: if((stat=actionformatx(&dumpoptions.actions[i]))) goto done; break; + case ACT_READ: if((stat=actionread(&dumpoptions.actions[i]))) goto done; break; + case ACT_WRITE: if((stat=actionwrite(&dumpoptions.actions[i]))) goto done; break; + } + } + +done: + fflush(stdout); + if(stat) + fprintf(stderr,"fail: %s\n",nc_strerror(stat)); + return (stat ? 1 : 0); +}