Extend the netcdf API to support programmatic changes to the plugin search path

CMakeLists.txt

@@ -78,6 +78,10 @@ add_library(netCDF::netcdf ALIAS netcdf)
 # configure.ac.
 set(NC_DISPATCH_VERSION 5)
 
+# Version of the plugin path dispatch table. This must match the value in
+# configure.ac.
+set(NC_PLUGINPATH_DISPATCH_VERSION 1)
+
 # Get system configuration, Use it to determine osname, os release, cpu. These
 # will be used when committing to CDash.
 find_program(UNAME NAMES uname)
@@ -869,6 +873,7 @@ if(NETCDF_ENABLE_TESTS)
   ###
   option(NETCDF_ENABLE_BENCHMARKS "Run benchmark Tests." OFF)
 
+  set(BUILD_BENCHMARKS ${NETCDF_ENABLE_BENCHMARKS} CACHE BOOL "alias for NETCDF_ENABLE_BENCHMARKS")
 
   ###
   # End known-failures.

RELEASE_NOTES.md

@@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.9.3 - TBD
 
+* Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #3024](https://github.com/Unidata/netcdf-c/pull/3024) for more information.
+
 ## Known Issue
 
 * Parallel operation using `mpich 4.2.0` (the default on `Ubuntu 24.04`) results in 'unexpected results' when running `nc_test4/run_par_test.sh`.  This can be fixed by removing `mpich` and associated libraries and development packages and installing `mpich 4.2.2` by hand, or by using `openmpi` provided via `apt`.

cmake/dependencies.cmake

@@ -226,12 +226,13 @@ if(USE_HDF5)
 endif(USE_HDF5)
 
 ################################
-# Curl Libraryies
+# Curl Libraries
 # Only needed for DAP (DAP2 or DAP4)
-# and NCZARR with S3 Support
+# and NCZARR S3 support
+# and byterange support
 ################################
 
-if( (NETCDF_ENABLE_DAP AND (NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT)) OR (NETCDF_ENABLE_NCZARR AND NETCDF_ENABLENCZARR_S3))
+if( NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT OR NETCDF_ENABLE_NCZARR_S3)
 
   # See if we have libcurl
   find_package(CURL)

config.h.cmake.in

@@ -493,9 +493,12 @@ with zip */
 /* Add extra properties to _NCProperties attribute */
 #cmakedefine NCPROPERTIES_EXTRA ${NCPROPERTIES_EXTRA}
 
-/* Idspatch table version */
+/* Version for dispatch table version */
 #cmakedefine NC_DISPATCH_VERSION ${NC_DISPATCH_VERSION}
 
+/* Version for plugin path dispatch table version */
+#cmakedefine NC_PLUGINPATH_DISPATCH_VERSION ${NC_PLUGINPATH_DISPATCH_VERSION}
+
 /* no IEEE float on this platform */
 #cmakedefine NO_IEEE_FLOAT 1
 

configure.ac

@@ -1153,7 +1153,7 @@ AC_ARG_ENABLE([benchmarks],
                               are timed. We use these tests to check netCDF performance.])])
 test "x$enable_benchmarks" = xyes || enable_benchmarks=no
 AC_MSG_RESULT($enable_benchmarks)
-if test "x$enable_HDF5" = xno -a "x$enable_benchmarks" = xyes; then
+if test "x$enable_hdf5" = xno -a "x$enable_benchmarks" = xyes; then
 AC_MSG_ERROR([Can't use benchmarks if HDF5 is disabled.])
 fi
 AM_CONDITIONAL(BUILD_BENCHMARKS, [test x$enable_benchmarks = xyes])
@@ -1633,6 +1633,7 @@ has_hdf5_ros3=no
 
 if test "x$enable_hdf5" = xyes; then
 
+   AC_DEFINE([NETCDF_ENABLE_HDF5], [1], [if true, use HDF5])
    AC_DEFINE([USE_HDF5], [1], [if true, use HDF5])
    AC_DEFINE([H5_USE_16_API], [1], [use HDF5 1.6 API])
 
@@ -2258,6 +2259,13 @@ AX_SET_META([NC_HAS_BZ2],[$have_bz2],[yes])
 AC_SUBST([NC_DISPATCH_VERSION], [5])
 AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.])
 
+# This is the version of the plugin path dispatch table.
+# If the dispatch table is changed, this should be incremented.
+# If this is changed, make sure the value in
+# CMakeLists.txt also changes to match.
+AC_SUBST([NC_PLUGINPATH_DISPATCH_VERSION], [1])
+AC_DEFINE_UNQUOTED([NC_PLUGINPATH_DISPATCH_VERSION], [${NC_PLUGINPATH_DISPATCH_VERSION}], [Plugin Path Dispatch table version.])
+
 #####
 # End netcdf_meta.h definitions.
 #####

include/Makefile.am

@@ -23,7 +23,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h		\
 ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h	\
 ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h	\
 hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h          \
-ncjson.h ncxml.h ncs3sdk.h
+ncjson.h ncxml.h ncs3sdk.h ncplugins.h
 
 if USE_DAP
 noinst_HEADERS += ncdap.h

include/hdf5internal.h

@@ -196,6 +196,11 @@ struct NC_HDF5_Filter {
     unsigned int* params;  /**< Params for arbitrary filter. */
 };
 
+/* The type for the NC_FORMATX_NC_HDF5 Global State Object */
+typedef struct GlobalHDF5 { /* libhdf5 dispatcher specific parameters */
+    int placeholder; /* ensure a non-empty struct */
+} GlobalHDF5;
+
 int NC4_hdf5_filter_initialize(void);
 int NC4_hdf5_filter_finalize(void);
 int NC4_hdf5_filter_remove(NC_VAR_INFO_T* var, unsigned int id);

include/nc4internal.h

@@ -107,7 +107,10 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
 /* Forward declarations. */
 struct NC_GRP_INFO;
 struct NC_TYPE_INFO;
+
+/* Opaque */
 struct NCRCinfo;
+struct NC_PluginPathDispatch;
 
 /**
  * This struct provides indexed Access to Meta-data objects. See the
@@ -461,15 +464,10 @@ extern int nc_get_alignment(int* thresholdp, int* alignmentp);
 /* Begin to collect global state info in one place (more to do) */
 
 typedef struct NCglobalstate {
-    int initialized;
     char* tempdir; /* track a usable temp dir */
     char* home; /* track $HOME */
     char* cwd; /* track getcwd */
     struct NCRCinfo* rcinfo; /* Currently only one rc file per session */
-    struct GlobalZarr { /* Zarr specific parameters */
-	char dimension_separator;
-	int default_zarrformat;
-    } zarr;
     struct GlobalAWS { /* AWS S3 specific parameters/defaults */
 	char* default_region;
 	char* config_file;
@@ -483,6 +481,12 @@ typedef struct NCglobalstate {
 	int alignment;
     } alignment;
     struct ChunkCache chunkcache;
+    /* Global dispatcher and states specific to each dispatcher
+       and indexed by NC_FORMATX */
+    struct FormatXGlobal {
+        void* state[NC_FORMATX_COUNT]; /* type is opaque (like e.g. file_info_format field) */
+        const struct NC_PluginPathDispatch** pluginapi; /*[NC_FORMATX_COUNT];*/
+    } formatxstate;
 } NCglobalstate;
 
 extern struct NCglobalstate* NC_getglobalstate(void);

include/ncdispatch.h

@@ -35,7 +35,7 @@
 
 /* Given a filename, check its magic number */
 /* Change magic number size from 4 to 8 to be more precise for HDF5 */
-#define MAGIC_NUMBER_LEN ((size_t)8)
+#define MAGIC_NUMBER_LEN ((unsigned long long)8)
 #define MAGIC_HDF5_FILE 1
 #define MAGIC_HDF4_FILE 2
 #define MAGIC_CDF1_FILE 1 /* std classic format */

include/ncplugins.h

@@ -0,0 +1,46 @@
+/*
+Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+See COPYRIGHT for license information.
+*/
+
+/*
+Common functionality for plugin paths/
+For internal use only.
+*/
+
+#ifndef NCPLUGINS_H
+#define NCPLUGINS_H
+
+/* Opaque */
+struct NClist;
+
+/* Define the plugin path management dispatch table */
+
+typedef struct NC_PluginPathDispatch {
+    int model; /* one of the NC_FORMATX #'s */
+    int dispatch_version;
+    int (*initialize)(void** statep, const struct NClist* initialpaths);
+    int (*finalize)(void** statep);
+    int (*read)(void* state, size_t* ndirsp, char** dirs);
+    int (*write)(void* state, size_t ndirs, char** const dirs);
+} NC_PluginPathDispatch;
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* Known Plugin Dispatchers */
+#ifdef USE_HDF5
+EXTERNL NC_PluginPathDispatch* NC4_hdf5_pluginpathtable;
+#endif
+#ifdef NETCDF_ENABLE_NCZARR
+EXTERNL NC_PluginPathDispatch* NCZ_pluginpathtable;
+#endif
+
+/* See the file netcdf_aux.h for plugin-related utility functions */
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /*NCPLUGINS_H*/

include/netcdf.h

@@ -226,10 +226,13 @@ Use this in mode flags for both nc_create() and nc_open(). */
 #define NC_FORMATX_NCZARR    (10)
 #define NC_FORMATX_UNDEFINED (0)
 
-  /* To avoid breaking compatibility (such as in the python library),
+/* Define the legal NC_FORMATX indices i.e. 0<=index<NC_FORMATX_COUNT */
+#define NC_FORMATX_COUNT ((NC_FORMATX_NCZARR)+1)
+
+/* To avoid breaking compatibility (such as in the python library),
    we need to retain the NC_FORMAT_xxx format as well. This may come
-  out eventually, as the NC_FORMATX is more clear that it's an extended
-  format specifier.*/
+   out eventually, as the NC_FORMATX is more clear that it's an extended
+   format specifier.*/
 
 #define NC_FORMAT_NC3       NC_FORMATX_NC3 /**< \deprecated As of 4.4.0, use NC_FORMATX_NC3 */
 #define NC_FORMAT_NC_HDF5   NC_FORMATX_NC_HDF5 /**< \deprecated As of 4.4.0, use NC_FORMATX_NC_HDF5 */

include/netcdf_aux.h

@@ -96,6 +96,70 @@ EXTERNL int ncaux_abort_compound(void* tag);
 EXTERNL int ncaux_add_field(void* tag,  const char *name, nc_type field_type,
 			   int ndims, const int* dimsizes);
 
+/**************************************************/
+/* Path-list Utilities */
+
+/**
+Parse a string into a sequence of path directories.
+
+The pathlist argument has the following syntax:
+    paths := <empty> | dirlist
+    dirlist := dir | dirlist separator dir
+    separator := ';' | ':'
+    dir := <OS specific directory path>
+
+@param pathlist a string encoding a list of directories
+@param sep  one of ';' | ':' | '\0' where '\0' means use the platform's default separator.
+@param ndirsp return the number of directories in dirsp
+@param dirsp return a vector of strings representing the directories parsed from pathlist; caller frees
+@return ::NC_NOERR
+
+Note that this function is called twice: first time to get the number of directories
+and second to get the directories.
+
+Author: Dennis Heimbigner
+*/
+
+EXTERNL int ncaux_plugin_path_parse(const char* pathlist, char sep, size_t* ndirsp, char** dirs);
+
+/**
+Concatenate a vector of directories with the separator between.
+This is more-or-less the inverse of the ncaux_plugin_path_parse function
+
+The resulting string has following syntax:
+    paths := <empty> | dirlist
+    dirlist := dir | dirlist separator dir
+    separator := ';' | ':'
+    dir := <OS specific directory path>
+
+@param ndirs the number of directories
+@param dirsp the directory vector to concatenate
+@param sep one of ';', ':', or '\0'
+@param catlen length of the cat arg including a nul terminator
+@param cat user provided space for holding the concatenation; nul termination guaranteed if catlen > 0.
+@return ::NC_NOERR
+@return ::NC_EINVAL for illegal arguments
+
+Note that this function is called twice: first time to get the expected size of
+the concatenated string and second to get the contents of the concatenation.
+
+Author: Dennis Heimbigner
+*/
+
+EXTERNL int ncaux_plugin_path_tostring(size_t ndirs, char** const dirs, char sep, size_t* catlen, char* cat);
+
+
+/*
+Reclaim a char** object possibly produced by ncaux_plugin_parse function.
+
+@param veclen the number of entries in vec
+@param vec    a char** vectore
+@return ::NC_NOERR
+@return ::NC_EINVAL for illegal arguments
+*/
+
+EXTERNL int ncaux_plugin_path_freestringvec(size_t veclen, char** vec);
+
 #if defined(__cplusplus)
 }
 #endif

include/netcdf_filter.h

@@ -12,8 +12,8 @@
 #ifndef NETCDF_FILTER_H
 #define NETCDF_FILTER_H 1
 
-/* API for libdispatch/dfilter.c
-*/
+/**************************************************/
+/* API for libdispatch/dfilter.c */
 
 /* Must match values in <H5Zpublic.h> */
 #ifndef H5Z_FILTER_DEFLATE
@@ -110,9 +110,65 @@ EXTERNL int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *leve
 EXTERNL int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle);
 EXTERNL int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep);
 
+/* Filter path query/set */
+EXTERNL int nc_filter_path_query(int id);
+
+/**************************************************/
+/* API for libdispatch/dplugin.c */
+
+/* Plugin path functions */
+
+/**
+ * This function is called as part of nc_initialize.
+ * Its purpose is to initialize the plugin paths state.
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+*/
+
+EXTERNL int nc_plugin_path_initialize(void);
+
+/**
+ * This function is called as part of nc_finalize()
+ * Its purpose is to clean-up plugin path state.
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+*/
+
+EXTERNL int nc_plugin_path_finalize(void);
+
+/**
+ * Return the current sequence of directories in the internal plugin path list.
+ * Since this function does not modify the plugin path, it can be called at any time.
+ * @param formatx specify which dispatch implementatio to read: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR.
+ * @param ndirsp return the number of dirs in the internal path list
+ * @param dirs memory for storing the sequence of directies in the internal path list.
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+ *
+ * As a rule, this function needs to be called twice.
+ * The first time with npaths not NULL and pathlist set to NULL
+ *     to get the size of the path list.
+ * The second time with pathlist not NULL to get the actual sequence of paths.
+*/
+
+EXTERNL int nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs);
+
+/**
+ * Empty the current internal path sequence
+ * and replace with the sequence of directories argument.
+ *
+ * Using a paths argument of NULL or npaths argument of 0 will clear the set of plugin paths.
+ * @param formatx specify which dispatch implementation to write: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR.
+ * @param ndirs length of the dirs argument
+ * @param dirs to overwrite the current internal path list
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+*/
+
+EXTERNL int nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs);
+
 #if defined(__cplusplus)
 }
 #endif
-/**************************************************/
 
 #endif /* NETCDF_FILTER_H */

include/netcdf_json.h

@@ -1086,7 +1086,6 @@ NCJinsertstring(NCjson* object, const char* key, const char* value)
     else
         NCJnewstring(NCJ_STRING,value,&jvalue);
     NCJinsert(object,key,jvalue);
-done:
     return NCJTHROW(stat);
 }
 
@@ -1100,7 +1099,6 @@ NCJinsertint(NCjson* object, const char* key, long long ivalue)
     snprintf(digits,sizeof(digits),"%lld",ivalue);
     NCJnewstring(NCJ_STRING,digits,&jvalue);
     NCJinsert(object,key,jvalue);
-done:
     return NCJTHROW(stat);
 }
 
@@ -1305,6 +1303,8 @@ netcdf_supresswarnings(void)
     ignore = (void*)NCJparse;
     ignore = (void*)NCJdump;
     ignore = (void*)NCJtotext;
+    ignore = (void*)NCJinsertstring;
+    ignore = (void*)NCJinsertint;
     ignore = ignore;
 }
 #endif /*NETCDF_JSON_H*/