Merge branch 'release/3.2' into cbl-5865

C/Cpp_include/c4Database.hh

@@ -50,10 +50,12 @@ struct C4Database
 
     using Config = C4DatabaseConfig2;
 
-    /** Registers a directory path to load extension libraries from, such as Vector Search.
-        Must be called before opening a database that will use an extension. */
+    // Deprecated in favor of enableExtension!
     static void setExtensionPath(slice path);
 
+    /** Attempts to discover and verify the named extension in the provided path */
+    static void enableExtension(slice name, slice path);
+
     static bool exists(slice name, slice inDirectory);
     static void copyNamed(slice sourcePath, slice destinationName, const Config&);
     static bool deleteNamed(slice name, slice inDirectory);

C/Cpp_include/c4Index.hh

@@ -33,20 +33,31 @@ struct C4Index
 
     slice getName() const noexcept { return _name; }
 
+    C4IndexType     getType() const noexcept;
+    C4QueryLanguage getQueryLanguage() const noexcept;
+    slice           getExpression() const noexcept;
+
+    /// Writes the index options to `opts` and returns true. If there are none, returns false.
+    [[nodiscard]] bool getOptions(C4IndexOptions& opts) const noexcept;
+
 #ifdef COUCHBASE_ENTERPRISE
+    bool isTrained() const;
+
     /// Finds new or updated documents for which vectors need to be recomputed by the application.
     /// If there are none, returns NULL.
     /// @param limit  The maximum number of documents/vectors to return. If this is less than
     ///               the total number, the rest will be returned on the next call to `beginUpdate`.
     /// @warning  Do not call `beginUpdate` again until you're done with the returned updater;
     ///           it's not valid to have more than one update in progress at a time.
-    Retained<struct C4IndexUpdater> beginUpdate(size_t limit);
+    Retained<C4IndexUpdater> beginUpdate(size_t limit);
 #endif
 
   protected:
     friend class litecore::CollectionImpl;
     static Retained<C4Index> getIndex(C4Collection*, slice name);
 
+    C4Index(C4Collection* coll, std::string name) : _collection(coll), _name(std::move(name)) {}
+
     Retained<C4Collection> _collection;
     std::string            _name;
 };

C/c4.exp

@@ -402,6 +402,12 @@ _c4_dumpInstances
 _gC4ExpectExceptions
 
 _c4_setExtensionPath
+_c4_enableExtension
+
+_c4index_getType
+_c4index_getQueryLanguage
+_c4index_getExpression
+_c4index_getOptions
 
 _FLDoc_FromJSON
 _FLDoc_Retain

C/c4CAPI.cc

@@ -374,6 +374,10 @@ C4SliceResult c4coll_getIndexesInfo(C4Collection* coll, C4Error* C4NULLABLE outE
 
 void c4_setExtensionPath(C4String path) noexcept { C4Database::setExtensionPath(path); }
 
+bool c4_enableExtension(C4String name, C4String extensionPath, C4Error* outError) noexcept {
+    return tryCatch(outError, [=] { C4Database::enableExtension(name, extensionPath); });
+}
+
 bool c4db_exists(C4String name, C4String inDirectory) noexcept { return C4Database::exists(name, inDirectory); }
 
 bool c4key_setPassword(C4EncryptionKey* outKey, C4String password, C4EncryptionAlgorithm alg) noexcept {
@@ -553,7 +557,7 @@ bool c4db_createIndex2(C4Database* database, C4Slice name, C4Slice indexSpec, C4
 }
 
 bool c4coll_isIndexTrained(C4Collection* collection, C4Slice name, C4Error* outError) noexcept {
-    memset(outError, 0, sizeof(C4Error));
+    if ( outError ) *outError = kC4NoError;
     return tryCatch(outError, [=] { return collection->isIndexTrained(name); });
 }
 
@@ -901,6 +905,22 @@ C4Document* c4enum_getDocument(C4DocEnumerator* e, C4Error* outError) noexcept {
     });
 }
 
+#pragma mark - INDEXES:
+
+C4IndexType c4index_getType(C4Index* index) C4API { return index->getType(); }
+
+C4QueryLanguage c4index_getQueryLanguage(C4Index* index) C4API { return index->getQueryLanguage(); }
+
+C4String c4index_getExpression(C4Index* index) C4API { return index->getExpression(); }
+
+bool c4index_getOptions(C4Index* index, C4IndexOptions* outOpts) C4API { return index->getOptions(*outOpts); }
+
+#ifdef COUCHBASE_ENTERPRISE
+bool c4index_isTrained(C4Index* index, C4Error* C4NULLABLE outError) C4API {
+    return c4coll_isIndexTrained(index->getCollection(), index->getName(), outError);
+}
+#endif
+
 #pragma mark - OBSERVERS:
 
 C4DatabaseObserver* c4dbobs_createOnCollection(C4Collection* coll, C4CollectionObserverCallback callback,

C/c4Database.cc

@@ -62,6 +62,10 @@ C4EncryptionKey C4EncryptionKeyFromPasswordSHA1(slice password, C4EncryptionAlgo
 
 void C4Database::setExtensionPath(slice path) { SQLiteDataFile::setExtensionPath(string(path)); }
 
+void C4Database::enableExtension(slice name, slice path) {
+    SQLiteDataFile::enableExtension(string(name), string(path));
+}
+
 #pragma mark - STATIC LIFECYCLE METHODS:
 
 static FilePath dbPath(slice name, slice parentDir) {

C/c4Index.cc

@@ -28,9 +28,70 @@ using namespace fleece;
 using namespace litecore;
 
 struct C4IndexImpl final : public C4Index {
-    C4IndexImpl(C4Collection* c, slice name) : _spec(asInternal(c)->keyStore().getIndex(name)) {
-        _collection = c;
-        _name       = name;
+    C4IndexImpl(C4Collection* c, IndexSpec spec) : C4Index(c, spec.name), _spec(std::move(spec)) {}
+
+    C4IndexType getType() const noexcept { return C4IndexType(_spec.type); }
+
+    C4QueryLanguage getQueryLanguage() const noexcept { return C4QueryLanguage(_spec.queryLanguage); }
+
+    slice getExpression() const noexcept { return _spec.expression; }
+
+    bool getOptions(C4IndexOptions& opts) const noexcept {
+        opts = {};
+        if ( auto ftsOpts = _spec.ftsOptions() ) {
+            opts.language         = ftsOpts->language;
+            opts.ignoreDiacritics = ftsOpts->ignoreDiacritics;
+            opts.disableStemming  = ftsOpts->disableStemming;
+            opts.stopWords        = ftsOpts->stopWords;
+            return true;
+
+#ifdef COUCHBASE_ENTERPRISE
+        } else if ( auto vecOpts = _spec.vectorOptions() ) {
+            opts.vector.dimensions      = vecOpts->dimensions;
+            opts.vector.metric          = C4VectorMetricType(int(vecOpts->metric) + 1);
+            opts.vector.clustering.type = C4VectorClusteringType(vecOpts->clusteringType());
+            switch ( vecOpts->clusteringType() ) {
+                case vectorsearch::ClusteringType::Flat:
+                    {
+                        auto flat = std::get<vectorsearch::FlatClustering>(vecOpts->clustering);
+                        opts.vector.clustering.flat_centroids = flat.numCentroids;
+                        break;
+                    }
+                case vectorsearch::ClusteringType::MultiIndex:
+                    {
+                        auto multi = std::get<vectorsearch::MultiIndexClustering>(vecOpts->clustering);
+                        opts.vector.clustering.multi_bits          = multi.bitsPerSub;
+                        opts.vector.clustering.multi_subquantizers = multi.subquantizers;
+                        break;
+                    }
+            }
+            opts.vector.encoding.type = C4VectorEncodingType(vecOpts->encodingType());
+            switch ( vecOpts->encodingType() ) {
+                case vectorsearch::EncodingType::None:
+                    break;
+                case vectorsearch::EncodingType::PQ:
+                    {
+                        auto pq                               = std::get<vectorsearch::PQEncoding>(vecOpts->encoding);
+                        opts.vector.encoding.pq_subquantizers = pq.subquantizers;
+                        opts.vector.encoding.bits             = pq.bitsPerSub;
+                        break;
+                    }
+                case vectorsearch::EncodingType::SQ:
+                    {
+                        auto sq                   = std::get<vectorsearch::SQEncoding>(vecOpts->encoding);
+                        opts.vector.encoding.bits = sq.bitsPerDimension;
+                        break;
+                    }
+            }
+            if ( vecOpts->probeCount ) opts.vector.numProbes = *vecOpts->probeCount;
+            if ( vecOpts->minTrainingCount ) opts.vector.minTrainingSize = unsigned(*vecOpts->minTrainingCount);
+            if ( vecOpts->maxTrainingCount ) opts.vector.maxTrainingSize = unsigned(*vecOpts->maxTrainingCount);
+            opts.vector.lazy = vecOpts->lazyEmbedding;
+            return true;
+#endif
+        } else {
+            return false;
+        }
     }
 
 #ifdef COUCHBASE_ENTERPRISE
@@ -43,21 +104,35 @@ struct C4IndexImpl final : public C4Index {
     }
 #endif
 
-    optional<IndexSpec>           _spec;
+    IndexSpec                     _spec;
     Retained<litecore::LazyIndex> _lazy;
 };
 
-inline C4IndexImpl* asInternal(C4Index* index) { return static_cast<C4IndexImpl*>(index); }
+inline C4IndexImpl* asInternal(C4Index* i) { return static_cast<C4IndexImpl*>(i); }
 
-Retained<C4Index> C4Index::getIndex(C4Collection* c, slice name) {
-    Retained<C4IndexImpl> index = new C4IndexImpl(c, name);
-    if ( !index->_spec ) index = nullptr;
-    return index;
+inline C4IndexImpl const* asInternal(C4Index const* i) { return static_cast<C4IndexImpl const*>(i); }
+
+/*static*/ Retained<C4Index> C4Index::getIndex(C4Collection* c, slice name) {
+    if ( optional<IndexSpec> spec = asInternal(c)->keyStore().getIndex(name) ) {
+        return new C4IndexImpl(c, *std::move(spec));
+    } else {
+        return nullptr;
+    }
 }
 
+C4IndexType C4Index::getType() const noexcept { return asInternal(this)->getType(); }
+
+C4QueryLanguage C4Index::getQueryLanguage() const noexcept { return asInternal(this)->getQueryLanguage(); }
+
+slice C4Index::getExpression() const noexcept { return asInternal(this)->getExpression(); }
+
+bool C4Index::getOptions(C4IndexOptions& opts) const noexcept { return asInternal(this)->getOptions(opts); }
+
 
 #ifdef COUCHBASE_ENTERPRISE
 
+bool C4Index::isTrained() const { return _collection->isIndexTrained(_name); }
+
 Retained<C4IndexUpdater> C4Index::beginUpdate(size_t limit) { return asInternal(this)->beginUpdate(limit); }
 
 C4IndexUpdater::C4IndexUpdater(Retained<litecore::LazyIndexUpdate> u, C4Collection* c)

C/c4_ee.exp

@@ -443,6 +443,12 @@ _c4_dumpInstances
 _gC4ExpectExceptions
 
 _c4_setExtensionPath
+_c4_enableExtension
+
+_c4index_getType
+_c4index_getQueryLanguage
+_c4index_getExpression
+_c4index_getOptions
 
 _FLDoc_FromJSON
 _FLDoc_Retain
@@ -480,7 +486,9 @@ _c4keypair_privateKeyData
 _c4keypair_publicKeyData
 _c4keypair_publicKeyDigest
 
+_c4index_isTrained
 _c4index_beginUpdate
+
 _c4indexupdater_count
 _c4indexupdater_valueAt
 _c4indexupdater_setVectorAt

C/include/c4Collection.h

@@ -13,7 +13,7 @@
 #pragma once
 #include "c4DatabaseTypes.h"
 #include "c4DocumentTypes.h"
-#include "c4IndexTypes.h"
+#include "fleece/Fleece.h"
 
 C4_ASSUME_NONNULL_BEGIN
 C4API_BEGIN_DECLS
@@ -26,6 +26,11 @@ C4API_BEGIN_DECLS
     Observer-related functions are in c4Observer.h:
     - c4dbobs_createOnCollection
     - c4docobs_createWithCollection
+    Index-related functions are in c4Index.h:
+    - c4coll_createIndex
+    - c4coll_deleteIndex
+    - c4coll_getIndex
+    - c4coll_getIndexesInfo
 */
 
 
@@ -263,53 +268,6 @@ CBL_CORE_API C4Timestamp c4coll_nextDocExpiration(C4Collection*) C4API;
 NODISCARD CBL_CORE_API int64_t c4coll_purgeExpiredDocs(C4Collection*, C4Error* C4NULLABLE) C4API;
 
 
-/** @} */
-/** \name Indexes
-    @{ */
-
-
-/** Creates a collection index, of the values of specific expressions across all documents.
-    The name is used to identify the index for later updating or deletion; if an index with the
-    same name already exists, it will be replaced unless it has the exact same expressions.
-
-    The `indexSpec` argument is an expression, relative to a document, that describes what to index.
-    It can be in either the JSON query schema, or in N1QL syntax. It usually names a property,
-    but may also be a computed value based on properties.
-
-    @param collection  The collection to index.
-    @param name  The name of the index. Any existing index with the same name will be replaced,
-                 unless it has the identical expressions (in which case this is a no-op.)
-    @param indexSpec  The definition of the index in JSON or N1QL form. (See above.)
-    @param queryLanguage  The language of `indexSpec`, either JSON or N1QL.
-    @param indexType  The type of index (value full-text, etc.)
-    @param indexOptions  Options for the index. If NULL, each option will get a default value.
-    @param outError  On failure, will be set to the error status.
-    @return  True on success, false on failure. */
-NODISCARD CBL_CORE_API bool c4coll_createIndex(C4Collection* collection, C4String name, C4String indexSpec,
-                                               C4QueryLanguage queryLanguage, C4IndexType indexType,
-                                               const C4IndexOptions* C4NULLABLE indexOptions,
-                                               C4Error* C4NULLABLE              outError) C4API;
-
-/** Returns an object representing an existing index. */
-CBL_CORE_API C4Index* C4NULLABLE c4coll_getIndex(C4Collection* collection, C4String name,
-                                                 C4Error* C4NULLABLE outError) C4API;
-
-/** Deletes an index that was created by `c4coll_createIndex`.
-    @param collection  The collection to index.
-    @param name The name of the index to delete
-    @param outError  On failure, will be set to the error status.
-    @return  True on success, false on failure. */
-NODISCARD CBL_CORE_API bool c4coll_deleteIndex(C4Collection* collection, C4String name,
-                                               C4Error* C4NULLABLE outError) C4API;
-
-/** Returns information about all indexes in the collection.
-    The result is a Fleece-encoded array of dictionaries, one per index.
-    Each dictionary has keys `"name"`, `"type"` (a `C4IndexType`), and `"expr"` (the source expression).
-    @param collection  The collection to check
-    @param outError  On failure, will be set to the error status.
-    @return  A Fleece-encoded array of dictionaries, or NULL on failure. */
-CBL_CORE_API C4SliceResult c4coll_getIndexesInfo(C4Collection* collection, C4Error* C4NULLABLE outError) C4API;
-
 /** @} */
 /** @} */  // end Collections group
 

C/include/c4Database.h

@@ -45,10 +45,21 @@ NODISCARD CBL_CORE_API bool c4key_setPassword(C4EncryptionKey* encryptionKey, C4
 NODISCARD CBL_CORE_API bool c4key_setPasswordSHA1(C4EncryptionKey* encryptionKey, C4String password,
                                                   C4EncryptionAlgorithm alg) C4API;
 
-/** Registers a directory path to load extension libraries from, such as Vector Search.
-    Must be called before opening a database that will use an extension. */
+// Deprecated in favor of c4_enableExtension
 CBL_CORE_API void c4_setExtensionPath(C4String path) C4API;
 
+/** Asks LiteCore to look for and validate the presence of an extension given the name
+ * of the extension and the path in which it is supposed to reside.  It makes an attempt
+ * to only check things that have the possibility of being corrected by the user (i.e.
+ * if there is a bug in the extension and it cannot load functionally that won't be caught)
+ * @param name The name of the extension (corresponds to the lower case of the filename
+ *             without the extension)
+ * @param extensionPath The path in which the extension should be found
+ * @param outError On failure, will store the error.
+ * @return True on success, false on failure
+ */
+CBL_CORE_API bool c4_enableExtension(C4String name, C4String extensionPath, C4Error* outError) C4API;
+
 /** @} */
 
 //////// DATABASE API:

C/include/c4Error.h

@@ -186,6 +186,12 @@ typedef struct C4Error {
 #endif
 } C4Error;
 
+#ifdef __cplusplus
+static constexpr C4Error kC4NoError = {};
+#else
+#    define kC4NoError ((C4Error){})
+#endif
+
 // C4Error C API: