From 189629a3b04315b1548a87292cca2abdd9bf5799 Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Wed, 11 Sep 2024 14:33:07 -0700 Subject: [PATCH 1/4] Expose DataFile::_log() as public --- LiteCore/Storage/DataFile.hh | 4 +++- LiteCore/Support/Logging.cc | 7 ------- LiteCore/Support/Logging.hh | 8 +++++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/LiteCore/Storage/DataFile.hh b/LiteCore/Storage/DataFile.hh index d145acea0..9d465f7ba 100644 --- a/LiteCore/Storage/DataFile.hh +++ b/LiteCore/Storage/DataFile.hh @@ -178,7 +178,9 @@ namespace litecore { void _logVerbose(const char* format, ...) const __printflike(2, 3) { LOGBODY(Verbose) } - void _logDebug(const char* format, ...) const __printflike(2, 3){LOGBODY(Debug)} + void _logDebug(const char* format, ...) const __printflike(2, 3) { LOGBODY(Debug) } + + void _log(LogLevel level, const char* format, ...) const __printflike(3, 4){LOGBODY_(level)} //////// SHARED OBJECTS: diff --git a/LiteCore/Support/Logging.cc b/LiteCore/Support/Logging.cc index d4bdbdb2b..adde707cc 100644 --- a/LiteCore/Support/Logging.cc +++ b/LiteCore/Support/Logging.cc @@ -689,13 +689,6 @@ namespace litecore { Assert(_domain.registerParentObject(getObjectRef(), parentObjRef)); } - void Logging::_log(LogLevel level, const char* format, ...) const { - va_list args; - va_start(args, format); - _logv(level, format, args); - va_end(args); - } - void Logging::_logv(LogLevel level, const char* format, va_list args) const { _domain.computeLevel(); if ( _domain.willLog(level) ) _domain.vlog(level, this, true, format, args); diff --git a/LiteCore/Support/Logging.hh b/LiteCore/Support/Logging.hh index 69f98e6c2..487cd0eb5 100644 --- a/LiteCore/Support/Logging.hh +++ b/LiteCore/Support/Logging.hh @@ -215,11 +215,12 @@ namespace litecore { virtual std::string loggingIdentifier() const; virtual std::string loggingClassName() const; -#define LOGBODY(LEVEL) \ +#define LOGBODY_(LEVEL) \ va_list args; \ va_start(args, format); \ - _logv(LogLevel::LEVEL, format, args); \ + _logv(LEVEL, format, args); \ va_end(args); +#define LOGBODY(LEVEL) LOGBODY_(LogLevel::LEVEL) void warn(const char* format, ...) const __printflike(2, 3) { LOGBODY(Warning) } @@ -234,7 +235,8 @@ namespace litecore { bool willLog(LogLevel level = LogLevel::Info) const { return _domain.willLog(level); } - void _log(LogLevel level, const char* format, ...) const __printflike(3, 4); + void _log(LogLevel level, const char* format, ...) const __printflike(3, 4) { LOGBODY_(level) } + void _logv(LogLevel level, const char* format, va_list) const __printflike(3, 0); // Add key=value pairs to the output. They are space separated. If output is not empty From 87fa09f6b0cb8d5fcaf6603a39054f51b17dc5ca Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Fri, 3 Nov 2023 11:39:51 -0700 Subject: [PATCH 2/4] Make sync work across VV upgrades The transition from tree-based revids to version vectors is tricky for replication, and the prior approach (adding a fake 'legacy' version when creating the vector) didn't turn out to work. Instead, when a doc is upgraded in memory to a VectorRecord it keeps its tree-based/legacy revids. When the doc is saved it creates a version vector for the local Revision but also remembers the prior legacy revid. Remote Revisions keep their tree-based IDs until VVs are pulled from or pushed to those remotes. Revision histories may now include one or more legacy revids at the end. These are what allow the recipient of a revision (an active or passive puller) to match up an incoming version vector with a local legacy revid. There's still a question of how long we need to keep the legacy revid of the document. It only needs to be used once per remote, but in a P2P scenario you don't know if you might have to sync with some new peer that hasn't upgraded that doc yet. In any case the revid isn't very big (~24 bytes) so it may not be worth removing. --- C/tests/c4DatabaseTest.cc | 222 ++++++------------- C/tests/c4Test.cc | 9 +- C/tests/c4Test.hh | 2 + LiteCore/Database/VectorDocument.cc | 189 +++++++++------- LiteCore/RevTrees/SourceID.hh | 4 + LiteCore/RevTrees/VectorRecord.cc | 194 ++++++---------- LiteCore/RevTrees/VectorRecord.hh | 55 ++--- LiteCore/RevTrees/VersionVector.cc | 28 +-- LiteCore/RevTrees/VersionVector.hh | 15 +- LiteCore/RevTrees/VersionVectorWithLegacy.hh | 162 ++++++++++++++ LiteCore/tests/VectorRecordTest.cc | 3 + Replicator/ReplicatorTypes.cc | 1 + Replicator/tests/ReplicatorVVUpgradeTest.cc | 60 ++--- Xcode/LiteCore.xcodeproj/project.pbxproj | 2 + 14 files changed, 496 insertions(+), 450 deletions(-) create mode 100644 LiteCore/RevTrees/VersionVectorWithLegacy.hh diff --git a/C/tests/c4DatabaseTest.cc b/C/tests/c4DatabaseTest.cc index ba2c4012a..0e0938d8a 100644 --- a/C/tests/c4DatabaseTest.cc +++ b/C/tests/c4DatabaseTest.cc @@ -1418,161 +1418,83 @@ N_WAY_TEST_CASE_METHOD(C4DatabaseTest, "Database Upgrade To Version Vectors", "[ auto defaultColl = c4db_getDefaultCollection(db, nullptr); - SECTION("Read-Only") { - // Check doc 1: - C4Document* doc; - doc = c4coll_getDoc(defaultColl, "doc-001"_sl, true, kDocGetAll, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "2-c001d00d"); - alloc_slice history(c4doc_getRevisionHistory(doc, 0, nullptr, 0)); - CHECK(history == "2-c001d00d"); - CHECK(doc->sequence == 7); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"one","rev":"two"})"); - c4doc_release(doc); - - // Check doc 2: - doc = c4coll_getDoc(defaultColl, "doc-002"_sl, true, kDocGetAll, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "3-deadbeef"); - history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(history == "3-deadbeef"); - CHECK(doc->sequence == 9); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"two","rev":"three"})"); - alloc_slice remoteVers = c4doc_getRemoteAncestor(doc, 1); - CHECK(remoteVers == "3-deadbeef"); - CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"two","rev":"three"})"); - c4doc_release(doc); - - // Check doc 3: - doc = c4coll_getDoc(defaultColl, "doc-003"_sl, true, kDocGetAll, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "3-deadbeef"); - history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(history == "3-deadbeef, 2-c001d00d"); - CHECK(doc->sequence == 11); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"three","rev":"three"})"); - remoteVers = c4doc_getRemoteAncestor(doc, 1); - CHECK(remoteVers == "2-c001d00d"); - CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"three","rev":"two"})"); - c4doc_release(doc); - - // Check doc 4: - doc = c4coll_getDoc(defaultColl, "doc-004"_sl, true, kDocGetAll, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "3-deadbeef"); - history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(history == "3-deadbeef"); // parent 2-c001d00d doesn't show up bc it isn't a remote - CHECK(doc->sequence == 14); - CHECK(doc->flags == (kDocConflicted | kDocExists)); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"four","rev":"three"})"); - remoteVers = c4doc_getRemoteAncestor(doc, 1); - CHECK(remoteVers == "3-cc"); - REQUIRE(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(history == "3-cc"); - CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"ans*wer":42})"); - c4doc_release(doc); - - // Check deleted doc: - doc = c4coll_getDoc(defaultColl, "doc-DEL"_sl, true, kDocGetAll, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "1-abcd"); - history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(history == "1-abcd"); - CHECK(doc->sequence == 6); - CHECK(doc->flags == (kDocDeleted | kDocExists)); - c4doc_release(doc); - } - - SECTION("Upgrading") { - // Note: The revID/version checks below hardcode the base timestamp used for upgrading legacy - // replicated revIDs. It's currently 0x1770000000000000 (see HybridClock.hh). If that value - // changes, or the scheme for converting rev-tree revIDs to versions changes, the values below - // need to change too. - REQUIRE(uint64_t(litecore::kMinValidTime) == 0x1770000000000000); - - // Check doc 1: - C4Document* doc; - doc = c4coll_getDoc(defaultColl, "doc-001"_sl, true, kDocGetUpgraded, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "1770000000000002@*"); - alloc_slice versionVector(c4doc_getRevisionHistory(doc, 0, nullptr, 0)); - CHECK(versionVector == "1770000000000002@*"); - CHECK(doc->sequence == 7); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"one","rev":"two"})"); - c4doc_release(doc); - - // Check doc 2: - doc = c4coll_getDoc(defaultColl, "doc-002"_sl, true, kDocGetUpgraded, ERROR_INFO()); - REQUIRE(doc); - CHECK(c4rev_getTimestamp(doc->revID) == uint64_t(litecore::kMinValidTime) + 3); - CHECK(slice(doc->revID) == "1770000000000003@?"); // 0x1770000000000003 = kMinValidTime + 3 - versionVector = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(versionVector == "1770000000000003@?"); - CHECK(doc->sequence == 9); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"two","rev":"three"})"); - alloc_slice remoteVers = c4doc_getRemoteAncestor(doc, 1); - CHECK(remoteVers == "1770000000000003@?"); - CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"two","rev":"three"})"); + // Check doc 1: + C4Document* doc; + doc = c4coll_getDoc(defaultColl, "doc-001"_sl, true, kDocGetAll, ERROR_INFO()); + REQUIRE(doc); + CHECK(slice(doc->revID) == "2-c001d00d"); + alloc_slice history(c4doc_getRevisionHistory(doc, 0, nullptr, 0)); + CHECK(history == "2-c001d00d"); + CHECK(doc->sequence == 7); + CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"one","rev":"two"})"); + c4doc_release(doc); + // Check doc 2: + doc = c4coll_getDoc(defaultColl, "doc-002"_sl, true, kDocGetAll, ERROR_INFO()); + REQUIRE(doc); + CHECK(slice(doc->revID) == "3-deadbeef"); + history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); + CHECK(history == "3-deadbeef"); + CHECK(doc->sequence == 9); + CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"two","rev":"three"})"); + alloc_slice remoteVers = c4doc_getRemoteAncestor(doc, 1); + CHECK(remoteVers == "3-deadbeef"); + CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); + CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"two","rev":"three"})"); + { // update & save it: - { - TransactionHelper t(db); - C4Document* newDoc = c4doc_update(doc, kFleeceBody, 0, ERROR_INFO()); - REQUIRE(newDoc); - CHECK(slice(newDoc->revID) > "178532e35bcd0001@*"); - alloc_slice newVersionVector(c4doc_getRevisionHistory(newDoc, 0, nullptr, 0)); - CHECK(newVersionVector.hasSuffix("@*; 1770000000000003@?")); - c4doc_release(newDoc); - } - c4doc_release(doc); + TransactionHelper t(db); + C4Document* newDoc = c4doc_update(doc, kFleeceBody, 0, ERROR_INFO()); + REQUIRE(newDoc); + CHECK(slice(newDoc->revID).hasSuffix("@*")); + CHECK(slice(newDoc->revID) > "178532e35bcd0001@*"); + alloc_slice newVersionVector(c4doc_getRevisionHistory(newDoc, 0, nullptr, 0)); + CHECK(newVersionVector.hasSuffix("@*; 3-deadbeef")); + c4doc_release(newDoc); + } + c4doc_release(doc); - // Check doc 3: - doc = c4coll_getDoc(defaultColl, "doc-003"_sl, true, kDocGetUpgraded, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "1770000000000003@*"); - versionVector = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(versionVector == "1770000000000003@*; 1770000000000002@?"); - CHECK(doc->sequence == 11); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"three","rev":"three"})"); - remoteVers = c4doc_getRemoteAncestor(doc, 1); - CHECK(remoteVers == "1770000000000002@?"); - CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"three","rev":"two"})"); - c4doc_release(doc); + // Check doc 3: + doc = c4coll_getDoc(defaultColl, "doc-003"_sl, true, kDocGetAll, ERROR_INFO()); + REQUIRE(doc); + CHECK(slice(doc->revID) == "3-deadbeef"); + history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); + CHECK(history == "3-deadbeef, 2-c001d00d"); + CHECK(doc->sequence == 11); + CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"three","rev":"three"})"); + remoteVers = c4doc_getRemoteAncestor(doc, 1); + CHECK(remoteVers == "2-c001d00d"); + CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); + CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"three","rev":"two"})"); + c4doc_release(doc); - // Check doc 4: - doc = c4coll_getDoc(defaultColl, "doc-004"_sl, true, kDocGetUpgraded, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "1770000000000003@*"); - versionVector = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(versionVector == "1770000000000003@*"); - CHECK(doc->sequence == 14); - CHECK(doc->flags == (kDocConflicted | kDocExists)); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"four","rev":"three"})"); - remoteVers = c4doc_getRemoteAncestor(doc, 1); - CHECK(remoteVers == "1770000000000003@?"); - REQUIRE(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - versionVector = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(versionVector == "1770000000000003@?"); - CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); - CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"ans*wer":42})"); - c4doc_release(doc); + // Check doc 4: + doc = c4coll_getDoc(defaultColl, "doc-004"_sl, true, kDocGetAll, ERROR_INFO()); + REQUIRE(doc); + CHECK(slice(doc->revID) == "3-deadbeef"); + history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); + CHECK(history == "3-deadbeef"); // parent 2-c001d00d doesn't show up bc it isn't a remote + CHECK(doc->sequence == 14); + CHECK(doc->flags == (kDocConflicted | kDocExists)); + CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"doc":"four","rev":"three"})"); + remoteVers = c4doc_getRemoteAncestor(doc, 1); + CHECK(remoteVers == "3-cc"); + REQUIRE(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); + history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); + CHECK(history == "3-cc"); + CHECK(c4doc_selectRevision(doc, remoteVers, true, WITH_ERROR())); + CHECK(Dict(c4doc_getProperties(doc)).toJSONString() == R"({"ans*wer":42})"); + c4doc_release(doc); - // Check deleted doc: - doc = c4coll_getDoc(defaultColl, "doc-DEL"_sl, true, kDocGetUpgraded, ERROR_INFO()); - REQUIRE(doc); - CHECK(slice(doc->revID) == "1770000000000001@*"); - versionVector = c4doc_getRevisionHistory(doc, 0, nullptr, 0); - CHECK(versionVector == "1770000000000001@*"); - CHECK(doc->sequence == 6); - CHECK(doc->flags == (kDocDeleted | kDocExists)); - c4doc_release(doc); - } + // Check deleted doc: + doc = c4coll_getDoc(defaultColl, "doc-DEL"_sl, true, kDocGetAll, ERROR_INFO()); + REQUIRE(doc); + CHECK(slice(doc->revID) == "1-abcd"); + history = c4doc_getRevisionHistory(doc, 0, nullptr, 0); + CHECK(history == "1-abcd"); + CHECK(doc->sequence == 6); + CHECK(doc->flags == (kDocDeleted | kDocExists)); + c4doc_release(doc); } // CBL-3706: Previously, calling these functions after deleting the default collection causes diff --git a/C/tests/c4Test.cc b/C/tests/c4Test.cc index 24e323ee0..2c7282a1e 100644 --- a/C/tests/c4Test.cc +++ b/C/tests/c4Test.cc @@ -248,10 +248,15 @@ void C4Test::closeDB() { db = nullptr; } -void C4Test::reopenDB() { - // Update _dbConfig in case db was reopened with different flags or encryption: +void C4Test::syncDBConfig() { + REQUIRE(db); _dbConfig.flags = c4db_getConfig2(db)->flags; _dbConfig.encryptionKey = c4db_getConfig2(db)->encryptionKey; +} + +void C4Test::reopenDB() { + // Update _dbConfig in case db was reopened with different flags or encryption: + syncDBConfig(); closeDB(); db = c4db_openNamed(kDatabaseName, &_dbConfig, ERROR_INFO()); diff --git a/C/tests/c4Test.hh b/C/tests/c4Test.hh index 44e45a2f8..78abde768 100644 --- a/C/tests/c4Test.hh +++ b/C/tests/c4Test.hh @@ -217,6 +217,8 @@ class C4Test { [[nodiscard]] const C4DatabaseConfig2& dbConfig() const { return _dbConfig; } + void syncDBConfig(); + [[nodiscard]] C4StorageEngine storageType() const { return _storage; } [[nodiscard]] bool isSQLite() const { return storageType() == kC4SQLiteStorageEngine; } diff --git a/LiteCore/Database/VectorDocument.cc b/LiteCore/Database/VectorDocument.cc index 23172b32f..34cee39af 100644 --- a/LiteCore/Database/VectorDocument.cc +++ b/LiteCore/Database/VectorDocument.cc @@ -13,6 +13,7 @@ #include "VectorDocument.hh" #include "VectorRecord.hh" #include "VersionVector.hh" +#include "VersionVectorWithLegacy.hh" #include "CollectionImpl.hh" #include "c4Database.hh" #include "DatabaseImpl.hh" @@ -196,25 +197,50 @@ namespace litecore { alloc_slice getRevisionHistory(unsigned maxRevs, const slice backToRevs[], unsigned backToRevsCount) const override { + alloc_slice result; if ( auto rev = _selectedRevision(); rev ) { - if ( _doc.versioning() == Versioning::Vectors ) { - VersionVector vers = rev->versionVector(); - if ( maxRevs > 0 && vers.count() > maxRevs ) vers.prune(maxRevs); - // Easter egg: if maxRevs is 0, don't replace '*' with my peer ID [tests use this] - return vers.asASCII(maxRevs ? mySourceID() : kMeSourceID); - } else { - string history = rev->revID.str(); + // First get the version vector of the selected revision: + VersionVecWithLegacy vvl(rev->revID); + if ( (backToRevsCount > 0 || _doc.lastLegacyRevID() || !vvl.legacy.empty()) && loadRevisions() ) { + // If current rev or peer have legacy revids, look for legacy ancestors: if ( _remoteID == RemoteID::Local ) { - if ( RemoteID parID = _doc.legacyTreeParent(); parID != RemoteID::Local ) { - auto parent = _doc.remoteRevision(parID); - if ( parent && parent->revID != rev->revID ) { history += ", " + parent->revID.str(); } + // Start with the doc's last legacy rev: + if ( revid lastRevID = _doc.lastLegacyRevID() ) vvl.legacy.emplace_back(lastRevID); + } + + // Search the remotes for earlier legacy revs that aren't conflicts: + unsigned curGen = UINT_MAX; + if ( !vvl.legacy.empty() ) curGen = revid(vvl.legacy.back()).generation(); + _doc.forAllRevs([&](RemoteID rem, Revision const& otherRev) { + if ( !otherRev.revID.isVersion() && otherRev.revID.generation() < curGen + && !(otherRev.flags & DocumentFlags::kConflicted) ) { + vvl.legacy.emplace_back(otherRev.revID); + } + }); + + // Sort legacy revs, remove dups, and stop after any revid in `backToRevs`: + vvl.sortLegacy(); + slice lastRev; + auto endBack = &backToRevs[backToRevsCount]; + for ( auto i = vvl.legacy.begin(); i != vvl.legacy.end(); ) { + if ( *i == lastRev ) { + i = vvl.legacy.erase(i); // remove duplicate + } else if ( std::find(backToRevs, endBack, *i) != endBack ) { + vvl.legacy.erase(i + 1, vvl.legacy.end()); // stop here + break; + } else { + lastRev = *i++; } } - return alloc_slice(history); } - } else { - return nullslice; + + // Finally convert to ASCII list. + // Easter egg: if maxRevs is 0, don't replace '*' with my peer ID [tests use this] + stringstream out; + vvl.write(out, maxRevs ? mySourceID() : kMeSourceID); + result = alloc_slice(out.str()); } + return result; } alloc_slice remoteAncestorRevID(C4RemoteID remote) override { @@ -263,7 +289,9 @@ namespace litecore { VersionVector _currentVersionVector() { auto curRevID = _doc.revID(); - return curRevID ? curRevID.asVersionVector() : VersionVector(); + if ( curRevID && curRevID.isVersion() ) return curRevID.asVersionVector(); + else + return VersionVector(); } static DocumentFlags convertNewRevisionFlags(C4RevisionFlags revFlags) { @@ -313,8 +341,6 @@ namespace litecore { // Handles `c4doc_put` when `rq.existingRevision` is false (a regular save.) // The caller has already done most of the checking, incl. MVCC. bool putNewRevision(const C4DocPutRequest& rq, C4Error* outError) override { - _doc.upgradeVersioning(); - // Update the flags: Revision newRev; newRev.flags = convertNewRevisionFlags(rq.revFlags); @@ -343,77 +369,53 @@ namespace litecore { // Handles `c4doc_put` when `rq.existingRevision` is true (called by the Pusher) int32_t putExistingRevision(const C4DocPutRequest& rq, C4Error* outError) override { - _doc.upgradeVersioning(); - - Revision newRev; - newRev.flags = convertNewRevisionFlags(rq.revFlags); - Doc fldoc = _newProperties(rq, outError); - if ( !fldoc ) return -1; - newRev.properties = fldoc.asDict(); + VersionVecWithLegacy curVers(_doc, RemoteID::Local); + VersionVecWithLegacy newVers((slice*)rq.history, rq.historyCount, mySourceID()); + auto remote = RemoteID(rq.remoteDBID); - // Parse the version vector from the history, and use it to update the db's clock: - VersionVector newVers; - newVers.readHistory((slice*)rq.history, rq.historyCount, mySourceID()); - if ( !newVers.updateClock(asInternal(database())->versionClock()) ) { + if ( !newVers.vector.updateClock(asInternal(database())->versionClock()) ) { if ( outError ) { - alloc_slice vecStr = newVers.asASCII(); + alloc_slice vecStr = newVers.vector.asASCII(); *outError = c4error_printf(LiteCoreDomain, kC4ErrorBadRevisionID, "Invalid timestamp in version vector %.*s", SPLAT(vecStr)); } return -1; } - alloc_slice newVersBinary = newVers.asBinary(); - newRev.revID = revid(newVersBinary); - - // Does it fit the current revision? - auto remote = RemoteID(rq.remoteDBID); - int commonAncestor = 1; - auto order = kNewer; - if ( _doc.exists() ) { - // See whether to update the local revision: - order = newVers.compareTo(_currentVersionVector()); - } + // Compare it with the current document revision: + versionOrder order = VersionVecWithLegacy::compare(newVers, curVers); + logPutExisting(curVers, newVers, order, remote); - // Log the update. Normally verbose, but a conflict is info (if from the replicator) - // or error (if local). - if ( DBLog.willLog(LogLevel::Verbose) || order == kConflicting ) { - static constexpr const char* kOrderName[4] = {"same", "older", "newer", "conflict"}; - alloc_slice newVersStr = newVers.asASCII(); - alloc_slice oldVersStr = _currentVersionVector().asASCII(); - if ( order != kConflicting ) - keyStore().dataFile()._logVerbose( - "putExistingRevision '%.*s' #%.*s ; currently #%.*s --> %s (remote %d)", SPLAT(_docID), - SPLAT(newVersStr), SPLAT(oldVersStr), kOrderName[order], rq.remoteDBID); - else if ( remote != RemoteID::Local ) - keyStore().dataFile()._logInfo("putExistingRevision '%.*s' #%.*s ; currently " - "#%.*s --> conflict (remote %d)", - SPLAT(_docID), SPLAT(newVersStr), SPLAT(oldVersStr), rq.remoteDBID); - else - keyStore().dataFile()._logWarning("putExistingRevision '%.*s' #%.*s ; " - "currently #%.*s --> conflict (remote %d)", - SPLAT(_docID), SPLAT(newVersStr), SPLAT(oldVersStr), - rq.remoteDBID); - } - - switch ( order ) { - case kSame: - case kOlder: - // I already have this revision, don't update local - commonAncestor = 0; - break; - case kNewer: - // It's newer, so update local to this revision: - _doc.setCurrentRevision(newRev); - break; - case kConflicting: - // Conflict, so update only the remote (if any): - if ( remote == RemoteID::Local ) { + // Check for no-op or conflict: + int commonAncestor = 1; + if ( order != kNewer ) { + if ( remote == RemoteID::Local ) { + if ( order == kConflicting ) { c4error_return(LiteCoreDomain, kC4ErrorConflict, nullslice, outError); return -1; + } else { + return 0; } - newRev.flags |= DocumentFlags::kConflicted; - break; + } + if ( order != kConflicting ) commonAncestor = 0; + } + + // Assemble a new Revision: + alloc_slice newVersBinary = newVers.vector.asBinary(); + Doc fldoc = _newProperties(rq, outError); + if ( !fldoc ) return -1; + Revision newRev; + newRev.properties = fldoc.asDict(); + newRev.revID = revid(newVersBinary); + newRev.flags = convertNewRevisionFlags(rq.revFlags); + + // Store the Revision into my VectorRecord: + if ( order == kNewer ) { + // It's newer, so update local to this revision: + _doc.setCurrentRevision(newRev); + } else { + // Conflict, so mark that and update only the remote: + newRev.flags |= DocumentFlags::kConflicted; } if ( remote != RemoteID::Local ) { @@ -430,6 +432,21 @@ namespace litecore { return commonAncestor; } + // Log the update. Normally verbose, but a conflict is info (if from the replicator) + // or error (if local). + void logPutExisting(VersionVecWithLegacy const& curVers, VersionVecWithLegacy const& newVers, + versionOrder order, RemoteID remote) { + LogLevel level = LogLevel::Verbose; + if ( order == kConflicting ) level = (remote == RemoteID::Local) ? LogLevel::Warning : LogLevel::Info; + if ( DBLog.willLog(level) ) { + static constexpr const char* kOrderName[4] = {"same", "older", "newer", "conflict"}; + stringstream out; + out << "putExistingRevision '" << _docID << "' [" << newVers << "]; currently [" << curVers << "] --> " + << kOrderName[order] << " (remote " << int(remote) << ")"; + keyStore().dataFile()._log(level, "%s", out.str().c_str()); + } + } + bool _saveIfRequested(const C4DocPutRequest& rq, C4Error* outError) { if ( rq.save && !save() ) { c4error_return(LiteCoreDomain, kC4ErrorConflict, nullslice, outError); @@ -579,18 +596,19 @@ namespace litecore { requestedVec.readASCII(revMap[rec.key], mySourceID); // Check whether the doc's current rev is this version, or a newer, or a conflict: - bool recUsesVVs = revid(rec.version).isVersion(); + versionOrder cmp; + bool recUsesVVs = revid(rec.version).isVersion(); if ( recUsesVVs ) { localVec.readBinary(rec.version); + cmp = localVec.compareTo(requestedVec); } else { - // Doc still has a legacy tree-based revID. Convert to a VV - localVec = VectorRecord::createLegacyVersionVector(rec); + // Doc still has a legacy tree-based revID + cmp = kOlder; } - auto cmp = localVec.compareTo(requestedVec); auto status = C4FindDocAncestorsResultFlags(cmp); // Check whether this revID matches any of the doc's remote revisions: - if ( remoteDBID != 0 ) { + if ( remoteDBID != 0 && recUsesVVs ) { VectorRecord::forAllRevIDs(rec, [&](RemoteID remote, revid aRev, bool hasBody) { if ( remote > RemoteID::Local && compareLocalRev(aRev) == kSame ) { if ( hasBody ) status |= kRevsHaveLocal; @@ -614,11 +632,14 @@ namespace litecore { delimiter delim(","); VectorRecord::forAllRevIDs(rec, [&](RemoteID, revid aRev, bool hasBody) { if ( delim.count() < maxAncestors && hasBody >= mustHaveBodies ) { - if ( !(compareLocalRev(aRev) & kNewer) ) { - alloc_slice vector = localVec.asASCII(mySourceID); - if ( added.insert(vector).second ) // [skip duplicate vectors] - result << delim << '"' << vector << '"'; + alloc_slice vector; + if ( recUsesVVs ) { + if ( !(compareLocalRev(aRev) & kNewer) ) vector = localVec.asASCII(mySourceID); + } else { + vector = aRev.expanded(); } + if ( vector && added.insert(vector).second ) // [skip duplicate vectors] + result << delim << '"' << vector << '"'; } }); diff --git a/LiteCore/RevTrees/SourceID.hh b/LiteCore/RevTrees/SourceID.hh index 7bfc4ec63..c68129b41 100644 --- a/LiteCore/RevTrees/SourceID.hh +++ b/LiteCore/RevTrees/SourceID.hh @@ -93,4 +93,8 @@ namespace litecore { kConflicting = kOlder | kNewer // The vectors conflict }; + static inline versionOrder mkOrder(bool newer, bool older) { + return versionOrder((newer ? kNewer : kSame) | (older ? kOlder : kSame)); + } + } // namespace litecore diff --git a/LiteCore/RevTrees/VectorRecord.cc b/LiteCore/RevTrees/VectorRecord.cc index faac62ecf..f131664d7 100644 --- a/LiteCore/RevTrees/VectorRecord.cc +++ b/LiteCore/RevTrees/VectorRecord.cc @@ -88,8 +88,11 @@ namespace litecore { // Keys in revision dicts (deliberately tiny and ineligible for SharedKeys, to save space.) static constexpr slice kRevPropertiesKey = "."; static constexpr slice kRevIDKey = "@"; + static constexpr slice kLegacyRevIDKey = "~"; static constexpr slice kRevFlagsKey = "&"; + bool Revision::hasVersionVector() const { return revID.isVersion(); } + Version Revision::version() const { return VersionVector::readCurrentVersionFromBinary(revID); } VersionVector Revision::versionVector() const { return VersionVector::fromBinary(revID); } @@ -99,10 +102,10 @@ namespace litecore { , _docID(rec.key()) , _sequence(rec.sequence()) , _subsequence(rec.subsequence()) - , _revID(rec.version()) + , _savedRevID(rec.version()) + , _revID(_savedRevID) , _docFlags(rec.flags()) - , _whichContent(rec.contentLoaded()) - , _versioning(Versioning::Vectors) { + , _whichContent(rec.contentLoaded()) { _current.revID = revid(_revID); _current.flags = _docFlags - (DocumentFlags::kConflicted | DocumentFlags::kSynced); if ( rec.exists() ) { @@ -171,7 +174,6 @@ namespace litecore { void VectorRecord::importRevTree(alloc_slice body, alloc_slice extra) { LogToAt(DBLog, Verbose, "VectorRecord: importing '%.*s' as RevTree", SPLAT(docID())); bool wasChanged = _changed; - _versioning = Versioning::RevTrees; _extraDoc = fleece::Doc(extra, kFLTrustedDontParse, sharedKeys()); RevTree revTree(body, extra, sequence()); const Rev* curRev = revTree.currentRevision(); @@ -205,60 +207,12 @@ namespace litecore { nuRev.flags = {}; if ( rev->flags & Rev::kDeleted ) nuRev.flags |= DocumentFlags::kDeleted; if ( rev->flags & Rev::kHasAttachments ) nuRev.flags |= DocumentFlags::kHasAttachments; + if ( rev->flags & Rev::kIsConflict ) nuRev.flags |= DocumentFlags::kConflicted; } setRemoteRevision(RemoteID(id), nuRev); } - // Determine which remote, if any, the current rev is based on: - _parentOfLocal = 0; - for ( const Rev* rev = curRev; rev && !_parentOfLocal; rev = rev->parent ) { - for ( auto [id, rem] : remoteRevMap ) { - if ( rem == rev ) { - _parentOfLocal = id; - break; - } - } - } - _changed = wasChanged; - - if ( _whichContent == kUpgrade ) upgradeVersioning(); - } - - // Changes old-style revIDs into Versions, and the document's revID to a VersionVector. - void VectorRecord::upgradeVersioning() { - if ( _versioning != Versioning::RevTrees ) return; - - LogToAt(DBLog, Verbose, "VectorRecord: upgrading '%.*s' to version vectors", SPLAT(docID())); - bool wasChanged = _changed; - - // Update current revID to a version vector with 1 or 2 components: - VersionVector vv; - Version meVers = Version::legacyVersion(revID(), kMeSourceID); - if ( _parentOfLocal > 0 ) { - auto parent = loadRemoteRevision(RemoteID(_parentOfLocal)); - vv.add(Version::legacyVersion(parent->revID, kLegacyRevSourceID)); - if ( revID() != parent->revID ) vv.add(meVers); - } else { - vv.add(meVers); - } - setRevID(revid(vv.asBinary())); - if ( DBLog.willLog(LogLevel::Verbose) ) { - LogToAt(DBLog, Verbose, "VectorRecord: '%.*s' revid is now %s", SPLAT(docID()), vv.asString().c_str()); - } - - // Update each remote revision's version: - RemoteID rid = loadNextRemoteID(RemoteID::Local); - while ( auto rev = loadRemoteRevision(rid) ) { - Version vers = Version::legacyVersion(rev->revID, kLegacyRevSourceID); - revidBuffer buf(vers); - rev->revID = buf.getRevID(); - setRemoteRevision(rid, rev); - rid = loadNextRemoteID(rid); - } - - _versioning = Versioning::Vectors; - _changed = wasChanged; } bool VectorRecord::loadData(ContentOption which) { @@ -273,7 +227,6 @@ namespace litecore { _whichContent = which; if ( which >= kCurrentRevOnly && oldWhich < kCurrentRevOnly ) readRecordBody(rec.body()); if ( which >= kEntireBody && oldWhich < kEntireBody ) readRecordExtra(rec.extra()); - if ( _versioning == Versioning::RevTrees && which == kUpgrade ) upgradeVersioning(); return true; } @@ -283,10 +236,10 @@ namespace litecore { rec.updateSequence(_sequence); rec.updateSubsequence(_subsequence); if ( _sequence > 0_seq ) rec.setExists(); - rec.setVersion(_revID); + rec.setVersion(_savedRevID); rec.setFlags(_docFlags); - if ( _bodyDoc ) rec.setBody(_bodyDoc.allocedData()); - if ( _extraDoc ) rec.setExtra(_extraDoc.allocedData()); + rec.setBody(_bodyDoc.allocedData()); + rec.setExtra(_extraDoc.allocedData()); rec.setContentLoaded(_whichContent); return rec; } @@ -343,6 +296,15 @@ namespace litecore { return nextRemoteID(remote); } + void VectorRecord::forAllRevs(const ForAllRevsCallback& callback) const { + RemoteID rem = RemoteID::Local; + optional rev; + while ( (rev = remoteRevision(rem)) ) { + callback(rem, *rev); + rem = nextRemoteID(rem); + } + } + // If _revisions is not mutable, makes a mutable copy and assigns it to _mutatedRevisions. void VectorRecord::mutateRevisions() { requireRemotes(); @@ -355,7 +317,6 @@ namespace litecore { // Returns the MutableDict for a revision. // If it's not mutable yet, replaces it with a mutable copy. MutableDict VectorRecord::mutableRevisionDict(RemoteID remote) { - Assert(remote > RemoteID::Local); mutateRevisions(); if ( _mutatedRevisions.count() <= int(remote) ) _mutatedRevisions.resize(int(remote) + 1); MutableDict revDict = _mutatedRevisions.getMutableDict(int(remote)); @@ -453,7 +414,7 @@ namespace litecore { if ( newRevID != _current.revID ) { _revID = alloc_slice(newRevID); _current.revID = revid(_revID); - _changed = _revIDChanged = true; + _changed = true; } } @@ -467,6 +428,11 @@ namespace litecore { } } + revid VectorRecord::lastLegacyRevID() const { + requireRemotes(); + return revid(_revisions[0].asDict()[kLegacyRevIDKey].asData()); + } + #pragma mark - CHANGE HANDLING: void VectorRecord::updateDocFlags() { @@ -527,23 +493,24 @@ namespace litecore { // If the revID hasn't been changed but the local properties have, generate a new revID: alloc_slice generatedRev; - if ( newRevision && !_revIDChanged ) { - switch ( _versioning ) { - case Versioning::RevTrees: - generatedRev = generateRevID(_current.properties, revID, flags); - break; - case Versioning::Vectors: - generatedRev = generateVersionVector(revID, versionClock); - } - revID = revid(generatedRev); + if ( newRevision && _revID == _savedRevID ) { + generatedRev = generateVersionVector(revID, versionClock); + revID = revid(generatedRev); setRevID(revID); - Log("Generated revID '%s'", revID.str().c_str()); + LogTo(DBLog, "Doc %.*s generated revID '%s'", FMTSLICE(_docID), revID.str().c_str()); + } + + Assert(revID.isVersion()); + if ( _savedRevID && !revid(_savedRevID).isVersion() ) { + LogToAt(DBLog, Verbose, "Doc %.*s saving legacy revID '%s'; new revID '%s'", FMTSLICE(_docID), + revid(_savedRevID).str().c_str(), revID.str().c_str()); + mutableRevisionDict(RemoteID::Local)[kLegacyRevIDKey].setData(_savedRevID); } alloc_slice body, extra; tie(body, extra) = encodeBodyAndExtra(); - bool updateSequence = (_sequence == 0_seq || _revIDChanged); + bool updateSequence = (_sequence == 0_seq || _revID != _savedRevID); Assert(revID); RecordUpdate rec(_docID, body, _docFlags); rec.version = revID; @@ -555,7 +522,8 @@ namespace litecore { _sequence = seq; _subsequence = updateSequence ? 0 : _subsequence + 1; - _changed = _revIDChanged = false; + _savedRevID = _revID; + _changed = false; // Update Fleece Doc to newly saved data: MutableDict mutableProperties = _current.properties.asMutable(); @@ -589,11 +557,17 @@ namespace litecore { } else { enc.beginArray(); DeDuplicateEncoder ddenc(enc); - // Write current rev: + // Write current rev. The body dict will be written first; then we snip that as the + // record's `body` property, and the encoder will write the rest of the `extra` with + // a back-pointer into the `body`. enc.beginDict(); enc.writeKey(kRevPropertiesKey); ddenc.writeValue(_current.properties, 1); body = alloc_slice(FLEncoder_Snip(enc)); + if ( revid legacy = lastLegacyRevID() ) { + enc.writeKey(kLegacyRevIDKey); + enc.writeData(legacy); + } enc.endDict(); // Write other revs: @@ -630,7 +604,7 @@ namespace litecore { void VectorRecord::dump(ostream& out) const { out << "\"" << (string)docID() << "\" #" << uint64_t(sequence()) << " "; - uint32_t nRevs = _revisions.count(); + uint32_t nRevs = std::max(_revisions.count(), uint32_t(1)); for ( uint32_t i = 0; i < nRevs; ++i ) { optional rev = remoteRevision(RemoteID(i)); if ( rev ) { @@ -647,6 +621,7 @@ namespace litecore { } } } + if ( _whichContent < kEntireBody ) out << "[other revs not loaded]"; } string VectorRecord::dump() const { @@ -719,62 +694,31 @@ namespace litecore { } /*static*/ void VectorRecord::forAllRevIDs(const RecordUpdate& rec, const ForAllRevIDsCallback& callback) { - if ( !revid(rec.version).isVersion() ) return forAllLegacyRevIDs(rec, callback); - - callback(RemoteID::Local, revid(rec.version), rec.body.size > 0); - if ( rec.extra.size > 0 ) { - fleece::impl::Scope scope(rec.extra, nullptr, rec.body); - Array remotes = ValueFromData(rec.extra, kFLTrusted).asArray(); - int n = 0; - for ( Array::iterator i(remotes); i; ++i, ++n ) { - if ( n > 0 ) { - Dict remote = i.value().asDict(); - if ( slice revID = remote[kRevIDKey].asData(); revID ) - callback(RemoteID(n), revid(revID), remote[kRevPropertiesKey] != nullptr); + if ( revid(rec.version).isVersion() ) { + callback(RemoteID::Local, revid(rec.version), rec.body.size > 0); + if ( rec.extra.size > 0 ) { + fleece::impl::Scope scope(rec.extra, nullptr, rec.body); + Array remotes = ValueFromData(rec.extra, kFLTrusted).asArray(); + int n = 0; + for ( Array::iterator i(remotes); i; ++i, ++n ) { + if ( n > 0 ) { + Dict remote = i.value().asDict(); + if ( slice revID = remote[kRevIDKey].asData(); revID ) + callback(RemoteID(n), revid(revID), remote[kRevPropertiesKey] != nullptr); + } } } - } - } - - /*static*/ void VectorRecord::forAllLegacyRevIDs(const RecordUpdate& rec, const ForAllRevIDsCallback& callback) { - RevTree revTree(rec.body, rec.extra, rec.sequence); - const Rev* curRev = revTree.currentRevision(); - bool foundCur = false; - for ( auto [id, rev] : revTree.remoteRevisions() ) { - auto vers = Version::legacyVersion(rev->revID, kLegacyRevSourceID); - callback(RemoteID(id), revidBuffer(vers).getRevID(), rev->isBodyAvailable()); - if ( rev == curRev ) foundCur = true; - } - // Finally the local version: - if ( !foundCur ) { - auto vers = Version::legacyVersion(curRev->revID, kMeSourceID); - callback(RemoteID::Local, revidBuffer(vers).getRevID(), curRev->isBodyAvailable()); - } - } - - // Given a record that hasn't been upgraded, synthesize a version vector for it. - /*static*/ VersionVector VectorRecord::createLegacyVersionVector(const RecordUpdate& rec) { - RevTree revTree(rec.body, rec.extra, rec.sequence); - const Rev* curRev = revTree.currentRevision(); - - // Look at the current rev's ancestry, finding the first that comes from a remote: - VersionVector vv; - bool curIsNew = true; - for ( const Rev* rev = curRev; rev && vv.empty(); rev = rev->parent ) { - for ( auto [id, rem] : revTree.remoteRevisions() ) { - (void)id; - if ( rem == rev ) { - // This rev came from a remote, so add a legacy version: - vv.add(Version::legacyVersion(rev->revID, kLegacyRevSourceID)); - if ( rev == curRev ) curIsNew = false; - } + } else { + // Legacy RevTree record: + RevTree revTree(rec.body, rec.extra, rec.sequence); + const Rev* curRev = revTree.currentRevision(); + // First the local version: + callback(RemoteID::Local, curRev->revID, curRev->isBodyAvailable()); + // Then the remotes: + for ( auto [id, rev] : revTree.remoteRevisions() ) { + if ( rev != curRev ) { callback(RemoteID(id), rev->revID, rev->isBodyAvailable()); } } } - if ( curIsNew ) { - // If the current rev didn't come from a remote, add a legacy version authored by me: - vv.add(Version::legacyVersion(rec.version, kMeSourceID)); - } - return vv; } } // namespace litecore diff --git a/LiteCore/RevTrees/VectorRecord.hh b/LiteCore/RevTrees/VectorRecord.hh index 56d18340a..7578010ac 100644 --- a/LiteCore/RevTrees/VectorRecord.hh +++ b/LiteCore/RevTrees/VectorRecord.hh @@ -40,6 +40,9 @@ namespace litecore { /// - kHasAttachments: Properties include references to blobs. DocumentFlags flags{}; + /// Returns true if `revID` is a version vector, false if it's a tree-based revid. + bool hasVersionVector() const; + /// Returns the current (first) version of the version vector encoded in the `revID`. [[nodiscard]] Version version() const; @@ -138,8 +141,6 @@ namespace litecore { /// The current revision's encoded Fleece data. slice currentRevisionData() const; - //---- Versioning: - /// The versioning system used by the document. Will be Vector unless the Record read /// from the db was still in rev-tree format. Versioning versioning() const FLPURE { return _versioning; } @@ -147,9 +148,9 @@ namespace litecore { /// Upgrades versioning from RevTrees to Vectors, in memory (doesn't save) void upgradeVersioning(); - /// If this doc uses RevTree versioning, this is the RemoteID that is the current - /// revision's closest ancestor. (If none is, returns Local.) - RemoteID legacyTreeParent() const { return RemoteID(_parentOfLocal); } + /// The document's last tree-based revid, before the first version-vectored rev was added. + /// (`nullslice` if it still has a legacy revid, or never had one.) + revid lastLegacyRevID() const FLPURE; //---- Modifying the document: @@ -218,13 +219,15 @@ namespace litecore { /// Same as \ref nextRemoteID, but loads the document's remote revisions if not in memory yet. RemoteID loadNextRemoteID(RemoteID); + using ForAllRevsCallback = function_ref; + + void forAllRevs(const ForAllRevsCallback&) const; + using ForAllRevIDsCallback = function_ref; /// Given only a record, find all the revision IDs and pass them to the callback. static void forAllRevIDs(const RecordUpdate&, const ForAllRevIDsCallback&); - static VersionVector createLegacyVersionVector(const RecordUpdate&); - //---- For testing: /// Generates a rev-tree revision ID given document properties, parent revision ID, and flags. @@ -258,26 +261,24 @@ namespace litecore { bool propertiesChanged() const; void clearPropertiesChanged() const; void updateDocFlags(); - static void forAllLegacyRevIDs(const RecordUpdate&, const ForAllRevIDsCallback&); - - KeyStore& _store; // The database KeyStore - FLEncoder _encoder{nullptr}; // Database shared Fleece Encoder - alloc_slice _docID; // The docID - sequence_t _sequence; // The Record's sequence - uint64_t _subsequence; // The Record's subsequence - DocumentFlags _docFlags; // Document-level flags - alloc_slice _revID; // Current revision ID backing store - Revision _current; // Current revision - fleece::RetainedValue _currentProperties; // Retains local properties - fleece::Doc _bodyDoc; // If saved, a Doc of the Fleece body - fleece::Doc _extraDoc; // Fleece Doc holding record `extra` - fleece::Array _revisions; // Top-level parsed body; stores revs - mutable fleece::MutableArray _mutatedRevisions; // Mutable version of `_revisions` - Versioning _versioning; // RevIDs or VersionVectors? - int _parentOfLocal{}; // (only used in imported revtree) - bool _changed{false}; // Set to true on explicit change - bool _revIDChanged{false}; // Has setRevID() been called? - ContentOption _whichContent; // Which parts of record are available + + KeyStore& _store; // The database KeyStore + FLEncoder _encoder{nullptr}; // Database shared Fleece Encoder + alloc_slice _docID; // The docID + sequence_t _sequence; // The Record's sequence + uint64_t _subsequence; // The Record's subsequence + DocumentFlags _docFlags; // Document-level flags + alloc_slice _savedRevID; // Revision ID saved in db (may == _revID) + alloc_slice _revID; // Current revision ID backing store + Revision _current; // Current revision + fleece::RetainedValue _currentProperties; // Retains local properties + fleece::Doc _bodyDoc; // If saved, a Doc of the Fleece body + fleece::Doc _extraDoc; // Fleece Doc holding record `extra` + fleece::Array _revisions; // Top-level parsed body; stores revs + mutable fleece::MutableArray _mutatedRevisions; // Mutable version of `_revisions` + Versioning _versioning; // RevIDs or VersionVectors? + bool _changed{false}; // Set to true on explicit change + ContentOption _whichContent; // Which parts of record are available // (Note: _changed doesn't reflect mutations to _properties; changed() checks for those.) }; } // namespace litecore diff --git a/LiteCore/RevTrees/VersionVector.cc b/LiteCore/RevTrees/VersionVector.cc index ce352690e..502644278 100644 --- a/LiteCore/RevTrees/VersionVector.cc +++ b/LiteCore/RevTrees/VersionVector.cc @@ -195,7 +195,7 @@ namespace litecore { return result; } - string VersionVector::asString() const { return std::string(asASCII()); } + string VersionVector::asString(SourceID myID) const { return std::string(asASCII(myID)); } void VersionVector::readASCII(slice str, SourceID mySourceID) { clear(); @@ -221,27 +221,6 @@ namespace litecore { return Version::readASCII(str); } - void VersionVector::readHistory(const slice history[], size_t historyCount, SourceID mySourceID) { - Assert(historyCount > 0); - readASCII(history[0], mySourceID); - if ( historyCount == 1 ) return; // -> Single version vector (or single version) - if ( count() > 1 ) - error::_throw(error::BadRevisionID, "Invalid version history (vector followed by other history)"); - if ( historyCount == 2 ) { - Version newVers = _vers[0]; - readASCII(history[1], mySourceID); - _add(newVers); // -> New version plus parent vector - } else { - for ( size_t i = 1; i < historyCount; ++i ) { - Version parentVers(history[i], mySourceID); - if ( auto time = timeOfAuthor(parentVers.author()); time == logicalTime::none ) - _vers.push_back(parentVers); - else if ( time <= parentVers.time() ) - error::_throw(error::BadRevisionID, "Invalid version history (increasing logicalTime)"); - } - } // -> List of versions - } - #pragma mark - OPERATIONS: versionOrder VersionVector::compareTo(const Version& v) const { @@ -330,9 +309,10 @@ namespace litecore { } void VersionVector::add(Version v) { - if ( auto t = timeOfAuthor(v.author()); t >= v.time() ) + if ( auto t = timeOfAuthor(v.author()); t > v.time() ) error::_throw(error::BadRevisionID, "New version timestamp is older than current one"); - _add(v); + else if ( t < v.time() ) + _add(v); } void VersionVector::_add(Version const& v) { diff --git a/LiteCore/RevTrees/VersionVector.hh b/LiteCore/RevTrees/VersionVector.hh index 25cec7bda..16471ec5e 100644 --- a/LiteCore/RevTrees/VersionVector.hh +++ b/LiteCore/RevTrees/VersionVector.hh @@ -75,14 +75,6 @@ namespace litecore { form, with no "*" allowed. mySourceID in the string will be changed to kMeSourceID (0). */ void readASCII(slice asciiString, SourceID mySourceID = kMeSourceID); - /** Assembles a version vector from its history, as a list of ASCII versions/vectors. - This can take a few forms: - - ["new version vector"] - - ["new version", "parent version vector"] - - ["new version", "parent version", "grandparent version" ...] - Throws BadRevisionID if the history isn't in a form it understands. */ - void readHistory(const slice history[], size_t historyCount, SourceID mySourceID = kMeSourceID); - /** Reads binary form. Overwrites any existing state. Throws BadRevisionID if the data's not valid.*/ void readBinary(slice binaryData); @@ -179,7 +171,7 @@ namespace litecore { [[nodiscard]] fleece::alloc_slice asASCII(SourceID myID = kMeSourceID) const; /// Same as \ref asASCII but returns a `std::string`, for convenience. - [[nodiscard]] std::string asString() const; + [[nodiscard]] std::string asString(SourceID myID = kMeSourceID) const; /// Writes vector in ASCII form to a slice-stream. /// If `myID` is given, occurrences of kMeSourceID will be written as that ID. @@ -281,8 +273,9 @@ namespace litecore { void validate() const; vec versionsBySource() const; - vec _vers; // versions, in order from latest to oldest. - size_t _nCurrent = 0; // Number of current/merged versions including the first + vec _vers; // versions, in order from latest to oldest. + size_t _nCurrent = 0; // Number of current/merged versions including the first + alloc_slice _revID; // legacy (tree-based) revision ID, if any }; } // namespace litecore diff --git a/LiteCore/RevTrees/VersionVectorWithLegacy.hh b/LiteCore/RevTrees/VersionVectorWithLegacy.hh new file mode 100644 index 000000000..cbecbb472 --- /dev/null +++ b/LiteCore/RevTrees/VersionVectorWithLegacy.hh @@ -0,0 +1,162 @@ +// +// VersionVectorWithLegacy.hh +// +// Copyright © 2023 Couchbase. All rights reserved. +// + +#pragma once +#include "VersionVector.hh" +#include "VectorRecord.hh" +#include "Error.hh" +#include +#include + +namespace litecore { + + /** A version vector and/or a legacy tree-based revision history; either or both may be missing. + If both are present, the legacy history is interpreted as older than the version vector. */ + class VersionVecWithLegacy { + public: + /// Constructs from a revid. + explicit VersionVecWithLegacy(revid revID) { + if ( revID ) { + if ( revID.isVersion() ) vector = VersionVector::fromBinary(revID); + else + legacy.emplace_back(revID); + } + } + + /// Constructs from a Revision of a VectorRecord. + explicit VersionVecWithLegacy(VectorRecord const& record, RemoteID remote) + : VersionVecWithLegacy(record.remoteRevision(remote).value().revID) { + if ( remote == RemoteID::Local && record.lastLegacyRevID() ) { + DebugAssert(legacy.empty()); + legacy.emplace_back(record.lastLegacyRevID()); + } + } + + /// Constructs from the history array of a C4DocPutRequest. + VersionVecWithLegacy(const slice history[], size_t historyCount, SourceID mySourceID) { + parse(history, historyCount, mySourceID); + } + + /// Constructs from an initializer list, for the convenience of unit tests. + VersionVecWithLegacy(std::initializer_list history, SourceID mySourceID) { + parse(history.begin(), history.size(), mySourceID); + } + + /// The version vector. May be empty. + VersionVector vector; + + + /// The legacy (tree-based) revid history, in reverse chronological order. May be empty. + /// Must be ordered by descending generation (see `sortLegacy()`.) + std::vector legacy; + + /// Sorts the `legacy` vector into canonical order (by descending generation.) + void sortLegacy() { + std::sort(legacy.begin(), legacy.end(), + [](auto& r1, auto& r2) { return revid(r1).generation() > revid(r2).generation(); }); + } + + /// Compares two VersionVecWithLegacy objects. + static versionOrder compare(VersionVecWithLegacy const& a, VersionVecWithLegacy const& b) { + extendedVersionOrder vectorOrder = extendedCompare(a.vector, b.vector); + extendedVersionOrder legacyOrder = extendedCompare(a.legacy, b.legacy); + if ( vectorOrder == kXConflicting || legacyOrder == kXConflicting ) return kConflicting; + else + return versionOrder(kVersionOrderMatrix[vectorOrder][legacyOrder]); + } + + /// Writes an ASCII representation to a stream. + void write(std::ostream& out, SourceID const& mySourceID) const { + out << vector.asString(mySourceID); + if ( !legacy.empty() ) { + std::string_view delimiter; + if ( !vector.empty() ) { + if ( vector.currentVersions() == vector.count() ) delimiter = "; "; + else + delimiter = ", "; + } + for ( alloc_slice const& rev : legacy ) { + out << delimiter << revid(rev).str(); + delimiter = ", "; + } + } + } + + friend std::ostream& operator<<(std::ostream& out, VersionVecWithLegacy const& ver) { + ver.write(out, kMeSourceID); + return out; + } + + private: + /// Common code in constructor to initialize from a history array. + void parse(const slice history[], size_t historyCount, SourceID mySourceID) { + // The last history item(s) may be legacy tree-based revids: + while ( historyCount > 0 ) { + revidBuffer lastHistory(history[historyCount - 1]); + if ( lastHistory.getRevID().isVersion() ) break; + legacy.emplace_back(lastHistory.getRevID()); + --historyCount; + } + std::reverse(legacy.begin(), legacy.end()); + + if ( historyCount == 1 ) { + vector.readASCII(history[0], mySourceID); // -> Single vector (or single version) + } else if ( historyCount == 2 ) { + vector.readASCII(history[1], mySourceID); + vector.add(Version(history[0])); // -> New version plus parent vector + } else if ( historyCount > 2 ) { + for ( ssize_t i = historyCount - 1; i >= 0; --i ) + vector.add(Version(history[i])); // -> List of versions + } + } + + enum extendedVersionOrder : uint8_t { + kXBothEmpty = 0, // Both are empty + kXOlderEmpty = 1, // A is older because it's empty, B isn't + kXNewerEmpty = 2, // A is newer because it's nonempty, B is empty + kXSame = 3 + kSame, // They're equal (both nonempty) + kXOlder = 3 + kOlder, // A is older (both nonempty) + kXNewer = 3 + kNewer, // A is newer (both nonempty) + kXConflicting = 3 + kConflicting, // They conflict (both nonempty) + }; + + // Table mapping the extendedVersionOrder of vectors and legacy into a versionOrder. + // `kTable[vec extOrder][legacy extOrder]` -> versionOrder + static constexpr uint8_t kVersionOrderMatrix[6][6] = { + {0, 1, 2, 0, 1, 2}, // both vectors empty + {1, 1, 3, 1, 1, 3}, // vector a is older because it's empty + {2, 3, 2, 2, 3, 2}, // vector a is newer because b is empty + {0, 3, 3, 0, 3, 3}, // vectors equal (nonempty) + {1, 3, 3, 1, 3, 3}, // vector a is older + {2, 3, 3, 2, 3, 3}, // vector a is newer + }; + + /// Compares two vectors or legacy histories, also taking into account whether either is empty. + template + static extendedVersionOrder extendedCompare(V const& a, V const& b) { + versionOrder emptyOrder = mkOrder(!a.empty(), !b.empty()); + if ( emptyOrder != kConflicting ) return extendedVersionOrder(emptyOrder); // -> one or both is empty + else + return extendedVersionOrder(3 + _compare(a, b)); // -> neither empty; do a proper compare + } + + static versionOrder _compare(VersionVector const& a, VersionVector const& b) { return a.compareTo(b); } + + /// Compares two legacy revision histories. They must be non-empty. + static versionOrder _compare(std::vector const& a, std::vector const& b) { + DebugAssert(!a.empty() && !b.empty()); + for ( auto iNew = a.begin(); iNew != a.end(); ++iNew ) { + auto iCur = std::find(b.begin(), b.end(), *iNew); + if ( iCur != b.end() ) { + // Found common rev. The history with newer (earlier) revs is newer: + return mkOrder(iNew != a.begin(), iCur != b.begin()); + } + } + return kConflicting; // no common revid at all + } + }; + +} // namespace litecore diff --git a/LiteCore/tests/VectorRecordTest.cc b/LiteCore/tests/VectorRecordTest.cc index 3d07a1d90..67f993bf9 100644 --- a/LiteCore/tests/VectorRecordTest.cc +++ b/LiteCore/tests/VectorRecordTest.cc @@ -63,6 +63,7 @@ N_WAY_TEST_CASE_METHOD(DataFileTestFixture, "Untitled VectorRecord", "[VectorRec CHECK(doc.sequence() == 0_seq); CHECK(doc.docID() == "Nuu"); CHECK(doc.revID() == nullslice); + CHECK(doc.lastLegacyRevID() == nullslice); CHECK(doc.flags() == DocumentFlags::kNone); Dict properties = doc.properties(); @@ -104,6 +105,7 @@ N_WAY_TEST_CASE_METHOD(DataFileTestFixture, "Save VectorRecord", "[VectorRecord] cerr << "Revisions: " << doc.revisionStorage() << "\n"; CHECK(doc.sequence() == 1_seq); CHECK(doc.revID().str() == "10000@*"); + CHECK(doc.lastLegacyRevID() == nullslice); CHECK(doc.flags() == DocumentFlags::kHasAttachments); CHECK(doc.properties().toJSON(true, true) == "{year:2525}"); CHECK(!doc.changed()); @@ -124,6 +126,7 @@ N_WAY_TEST_CASE_METHOD(DataFileTestFixture, "Save VectorRecord", "[VectorRecord] cerr << "Revisions: " << doc.revisionStorage() << "\n"; CHECK(doc.sequence() == 2_seq); CHECK(doc.revID().str() == "20000@*"); + CHECK(doc.lastLegacyRevID() == nullslice); CHECK(doc.flags() == DocumentFlags::kNone); CHECK(doc.properties().toJSON(true, true) == "{weekday:\"Friday\",year:2525}"); CHECK(!doc.changed()); diff --git a/Replicator/ReplicatorTypes.cc b/Replicator/ReplicatorTypes.cc index f9489361e..c434b93cb 100644 --- a/Replicator/ReplicatorTypes.cc +++ b/Replicator/ReplicatorTypes.cc @@ -109,6 +109,7 @@ namespace litecore::repl { history.reserve(10); history.push_back(revID); for ( const void *pos = historyBuf.buf, *end = historyBuf.end(); pos < end; ) { + while ( pos < end && *(char*)pos == ' ' ) pos = (char*)pos + 1; auto comma = slice(pos, end).findByteOrEnd(','); history.push_back(slice(pos, comma)); pos = comma + 1; diff --git a/Replicator/tests/ReplicatorVVUpgradeTest.cc b/Replicator/tests/ReplicatorVVUpgradeTest.cc index 1af56f6b1..aa7eb90c1 100644 --- a/Replicator/tests/ReplicatorVVUpgradeTest.cc +++ b/Replicator/tests/ReplicatorVVUpgradeTest.cc @@ -44,34 +44,16 @@ class ReplicatorVVUpgradeTest : public ReplicatorLoopbackTest { void upgrade() { upgrade(db, _collDB1); upgrade(db2, _collDB2); + syncDBConfig(); } }; -/* FIXME: This test is disabled because it exposes a design issue with version vector upgrades - and P2P replication. I need more time to think about a proper fix. - - The scenario is: - - Database A pushes docs to peer B. Both are still on rev-trees. - - A and B both upgrade to version vectors. - - A updates one of the docs it pushed. - - A pushes to B again. - This should succeed, but instead B reports a conflict. - - In database A, when the doc is upgraded by the replicator its version vector looks like - (`yyyy@AAAA`, xxxx@?), where `AAAA` is database A's UUID and `?` is the generic "pre-existing rev" - UUID. That's because database A knows the first revision exists elsewhere (it's marked with - the "remote #1" marker), while the second revision doesn't. - - However, in database B, the doc's version vector is (`xxxx@BBBB`). That's because database B - doesn't remember that the revision came from elsewhere. When a passive replicator saves - incoming revisions, it doesn't have a remote-ID to tag them with. That hasn't been an issue, - until now. - - So the upshot is that the version vector A sends conflicts with the version vector B has. - B sees that it has a version from AAAA, but is missing the version from BBBB. - --Jens - */ -TEST_CASE_METHOD(ReplicatorVVUpgradeTest, "Push After VV Upgrade", "[.disabled]") { +TEST_CASE_METHOD(ReplicatorVVUpgradeTest, "Push After VV Upgrade", "[Push]") { + //- db pushes docs to db2. Both are still on rev-trees. + //- db and db2 both upgrade to version vectors. + //- db updates two of the docs it pushed, and creates a new one. + //- db pushes to db2 again. + auto serverOpts = Replicator::Options::passive(_collSpec); importJSONLines(sFixturesDir + "names_100.json", _collDB1); @@ -83,11 +65,35 @@ TEST_CASE_METHOD(ReplicatorVVUpgradeTest, "Push After VV Upgrade", "[.disabled]" upgrade(); createNewRev(_collDB1, "0000001"_sl, kFleeceBody); createNewRev(_collDB1, "0000002"_sl, kFleeceBody); - _expectedDocumentCount = 2; + createNewRev(_collDB1, "newDoc"_sl, kFleeceBody); + _expectedDocumentCount = 3; Log("-------- Second Replication --------"); runReplicators(Replicator::Options::pushing(kC4OneShot, _collSpec), serverOpts); compareDatabases(); - validateCheckpoints(db, db2, "{\"local\":102}"); + validateCheckpoints(db, db2, "{\"local\":103}"); +} + +TEST_CASE_METHOD(ReplicatorVVUpgradeTest, "Pull After VV Upgrade", "[Pull]") { + //- db pushes docs to db2. Both are still on rev-trees. + //- db and db2 both upgrade to version vectors. + //- db updates two of the docs it pushed, and creates a new one. + //- db pushes to db2 again. + + importJSONLines(sFixturesDir + "names_100.json", _collDB1); + _expectedDocumentCount = 100; + Log("-------- First Replication --------"); + runPullReplication(); + + upgrade(); + createNewRev(_collDB1, "0000001"_sl, kFleeceBody); + createNewRev(_collDB1, "0000002"_sl, kFleeceBody); + createNewRev(_collDB1, "newDoc"_sl, kFleeceBody); + _expectedDocumentCount = 3; + + Log("-------- Second Replication --------"); + runPullReplication(); + + compareDatabases(); } diff --git a/Xcode/LiteCore.xcodeproj/project.pbxproj b/Xcode/LiteCore.xcodeproj/project.pbxproj index 939740b17..600aafa49 100644 --- a/Xcode/LiteCore.xcodeproj/project.pbxproj +++ b/Xcode/LiteCore.xcodeproj/project.pbxproj @@ -1422,6 +1422,7 @@ 27BEEE782A783A17005AD4BF /* VectorQueryTest.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = VectorQueryTest.cc; sourceTree = ""; }; 27C319EC1A143F5D00A89EDC /* KeyStore.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = KeyStore.cc; sourceTree = ""; }; 27C319ED1A143F5D00A89EDC /* KeyStore.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = KeyStore.hh; sourceTree = ""; }; + 27C3BD0C2AFEB1C800976E89 /* VersionVectorWithLegacy.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = VersionVectorWithLegacy.hh; sourceTree = ""; }; 27C77301216FCF5400D5FB44 /* c4PredictiveQueryTest+CoreML.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = "c4PredictiveQueryTest+CoreML.mm"; sourceTree = ""; }; 27CCC7D61E52613C00CE1989 /* Replicator.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Replicator.cc; sourceTree = ""; }; 27CCC7D71E52613C00CE1989 /* Replicator.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = Replicator.hh; sourceTree = ""; }; @@ -2522,6 +2523,7 @@ 274B36D025B271F7001FC28D /* Version.hh */, 27027CD1255F4A9B00A96D7D /* VersionVector.cc */, 27027CCD255F4A9B00A96D7D /* VersionVector.hh */, + 27C3BD0C2AFEB1C800976E89 /* VersionVectorWithLegacy.hh */, 276993E525390C3300FDF699 /* VectorRecord.cc */, 276993E125390C3300FDF699 /* VectorRecord.hh */, ); From d3d60fd1fa414fc2389eb6e2c1fb587644267a07 Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Fri, 17 Nov 2023 10:14:23 -0800 Subject: [PATCH 3/4] Removed unnecessary VV-upgrade constants Some constants added earlier for VV upgrading and dealing with legacy revids, which have been superseded by newer techniques, were nuked: - kDocGetUpgraded / kUpgrade - kLegacyRevSourceID - VectorRecord::_versioning --- C/include/c4DocumentTypes.h | 1 - LiteCore/RevTrees/RevTreeRecord.cc | 1 - LiteCore/RevTrees/SourceID.hh | 3 --- LiteCore/RevTrees/VectorRecord.hh | 8 -------- LiteCore/RevTrees/Version.cc | 14 -------------- LiteCore/Storage/Record.hh | 1 - LiteCore/tests/VersionVectorTest.cc | 19 ------------------- Replicator/DBAccess.cc | 2 +- Replicator/Pusher+Revs.cc | 2 +- Replicator/Pusher.cc | 2 +- 10 files changed, 3 insertions(+), 50 deletions(-) diff --git a/C/include/c4DocumentTypes.h b/C/include/c4DocumentTypes.h index f46635575..9902e90d6 100644 --- a/C/include/c4DocumentTypes.h +++ b/C/include/c4DocumentTypes.h @@ -51,7 +51,6 @@ typedef C4_ENUM(uint8_t, C4DocContentLevel){ kDocGetMetadata, ///< Only get revID and flags kDocGetCurrentRev, ///< Get current revision body but not other revisions/remotes kDocGetAll, ///< Get everything - kDocGetUpgraded, ///< Get everything, upgrade to latest format (version vectors) }; // Note: Same as litecore::ContentOption // Ignore warning about not initializing members, it must be this way to be C-compatible diff --git a/LiteCore/RevTrees/RevTreeRecord.cc b/LiteCore/RevTrees/RevTreeRecord.cc index 03bcb42d7..eb74a87ef 100644 --- a/LiteCore/RevTrees/RevTreeRecord.cc +++ b/LiteCore/RevTrees/RevTreeRecord.cc @@ -72,7 +72,6 @@ namespace litecore { switch ( _contentLoaded ) { case kEntireBody: - case kUpgrade: RevTree::decode(_rec.body(), _rec.extra(), _rec.sequence()); if ( auto cur = currentRevision(); cur && (_rec.flags() & DocumentFlags::kSynced) ) { // The kSynced flag is set when the document's current revision is pushed diff --git a/LiteCore/RevTrees/SourceID.hh b/LiteCore/RevTrees/SourceID.hh index c68129b41..dcf809609 100644 --- a/LiteCore/RevTrees/SourceID.hh +++ b/LiteCore/RevTrees/SourceID.hh @@ -81,9 +81,6 @@ namespace litecore { replication. */ constexpr SourceID kMeSourceID; - /** A `SourceID` used to represent a version created during upgrade of a database. */ - constexpr SourceID kLegacyRevSourceID{0x1e /*... rest is 00 */}; - /** The possible orderings of two Versions or VersionVectors. (Can be interpreted as two 1-bit flags.) */ enum versionOrder { diff --git a/LiteCore/RevTrees/VectorRecord.hh b/LiteCore/RevTrees/VectorRecord.hh index 7578010ac..a8ab76d89 100644 --- a/LiteCore/RevTrees/VectorRecord.hh +++ b/LiteCore/RevTrees/VectorRecord.hh @@ -141,13 +141,6 @@ namespace litecore { /// The current revision's encoded Fleece data. slice currentRevisionData() const; - /// The versioning system used by the document. Will be Vector unless the Record read - /// from the db was still in rev-tree format. - Versioning versioning() const FLPURE { return _versioning; } - - /// Upgrades versioning from RevTrees to Vectors, in memory (doesn't save) - void upgradeVersioning(); - /// The document's last tree-based revid, before the first version-vectored rev was added. /// (`nullslice` if it still has a legacy revid, or never had one.) revid lastLegacyRevID() const FLPURE; @@ -276,7 +269,6 @@ namespace litecore { fleece::Doc _extraDoc; // Fleece Doc holding record `extra` fleece::Array _revisions; // Top-level parsed body; stores revs mutable fleece::MutableArray _mutatedRevisions; // Mutable version of `_revisions` - Versioning _versioning; // RevIDs or VersionVectors? bool _changed{false}; // Set to true on explicit change ContentOption _whichContent; // Which parts of record are available // (Note: _changed doesn't reflect mutations to _properties; changed() checks for those.) diff --git a/LiteCore/RevTrees/Version.cc b/LiteCore/RevTrees/Version.cc index b4873d5a3..d7bd035ba 100644 --- a/LiteCore/RevTrees/Version.cc +++ b/LiteCore/RevTrees/Version.cc @@ -68,8 +68,6 @@ namespace litecore { bool SourceID::writeBinary(fleece::slice_ostream& out, bool current) const { uint8_t flag = current ? 0x80 : 0x00; if ( isMe() ) return out.writeByte(0 | flag); - else if ( *this == kLegacyRevSourceID ) - return out.writeByte(1 | flag) && out.writeByte(0x1e); else return out.writeByte(sizeof(_bytes) | flag) && out.write(&_bytes, sizeof(_bytes)); } @@ -81,13 +79,6 @@ namespace litecore { if ( len == 0 ) { *this = kMeSourceID; return true; - } else if ( len == 1 ) { - if ( uint8_t byte = in.readByte(); byte == 0x1e ) { - *this = kLegacyRevSourceID; - return true; - } else { - return false; - } } else { return len == sizeof(_bytes) && in.readAll(&_bytes, len); } @@ -146,9 +137,6 @@ namespace litecore { if ( in.peekByte() == '*' ) { in.readByte(); _author = kMeSourceID; - } else if ( in.peekByte() == '?' ) { - in.readByte(); - _author = kLegacyRevSourceID; } else { if ( !_author.readASCII(in.readAll(SourceID::kASCIILength)) ) return false; if ( _author.isMe() ) return false; @@ -167,8 +155,6 @@ namespace litecore { bool Version::writeASCII(slice_ostream& out, SourceID myID) const { if ( !out.writeHex(uint64_t(_time)) || !out.writeByte('@') ) return false; - else if ( _author == kLegacyRevSourceID ) - return out.writeByte('?'); else if ( auto& author = (_author.isMe()) ? myID : _author; author.isMe() ) return out.writeByte('*'); else diff --git a/LiteCore/Storage/Record.hh b/LiteCore/Storage/Record.hh index 459700ab3..c21b1bb47 100644 --- a/LiteCore/Storage/Record.hh +++ b/LiteCore/Storage/Record.hh @@ -50,7 +50,6 @@ namespace litecore { kMetaOnly, // Skip `extra` and `body` kCurrentRevOnly, // Skip `extra` kEntireBody, // Everything - kUpgrade, // Get everything, upgrade to latest format (version vectors) }; /** The unit of storage in a DataFile: a key, version and body (all opaque blobs); diff --git a/LiteCore/tests/VersionVectorTest.cc b/LiteCore/tests/VersionVectorTest.cc index b9ea86aa7..67edd0f0e 100644 --- a/LiteCore/tests/VersionVectorTest.cc +++ b/LiteCore/tests/VersionVectorTest.cc @@ -145,12 +145,6 @@ TEST_CASE("HybridClock", "[RevIDs]") { TEST_CASE("SourceID Binary", "[RevIDs]") { for ( const uint8_t& b : kMeSourceID.bytes() ) { CHECK(b == 0); } - uint8_t xb = 0x1e; - for ( const uint8_t& b : kLegacyRevSourceID.bytes() ) { - CHECK(b == xb); - xb = 0; - } - SourceID id; for ( size_t i = 0; i < sizeof(SourceID); ++i ) id.bytes()[i] = uint8_t(i + 1); CHECK(id != kMeSourceID); @@ -172,18 +166,6 @@ TEST_CASE("SourceID Binary", "[RevIDs]") { CHECK(id2 == kMeSourceID); CHECK(isCurrent == current); } - { - slice_ostream out(buf); - REQUIRE(kLegacyRevSourceID.writeBinary(out, current)); - slice result = out.output(); - CHECK(result.hexString() == (current ? "811e" : "011e")); - - slice_istream in(result); - REQUIRE(id2.readBinary(in, &isCurrent)); - CHECK(in.eof()); - CHECK(id2 == kLegacyRevSourceID); - CHECK(isCurrent == current); - } { slice_ostream out(buf); REQUIRE(id.writeBinary(out, current)); @@ -223,7 +205,6 @@ TEST_CASE("SourceID ASCII", "[RevIDs]") { TEST_CASE("Version", "[RevIDs]") { CHECK(Version(1_ht, kMeSourceID).asASCII() == "1@*"); - CHECK(Version(2_ht, kLegacyRevSourceID).asASCII() == "2@?"); Version v1(1_ht, Alice), v2(1_ht, Alice), v3(2_ht, Alice), v4(1_ht, Bob); CHECK(v1.time() == 1_ht); diff --git a/Replicator/DBAccess.cc b/Replicator/DBAccess.cc index f9e676ac3..4a28809e2 100644 --- a/Replicator/DBAccess.cc +++ b/Replicator/DBAccess.cc @@ -415,7 +415,7 @@ namespace litecore::repl { } Doc DBAccess::applyDelta(C4Collection* collection, slice docID, slice baseRevID, slice deltaJSON) { - Retained doc = getDoc(collection, docID, kDocGetUpgraded); + Retained doc = getDoc(collection, docID, kDocGetAll); if ( !doc ) error::_throw(error::NotFound); if ( !doc->selectRevision(baseRevID, true) || !doc->loadRevisionBody() ) return nullptr; return applyDelta(doc, deltaJSON, false); diff --git a/Replicator/Pusher+Revs.cc b/Replicator/Pusher+Revs.cc index aa291350d..04c9f91da 100644 --- a/Replicator/Pusher+Revs.cc +++ b/Replicator/Pusher+Revs.cc @@ -52,7 +52,7 @@ namespace litecore::repl { Dict root; auto collection = getCollection(); slice replacementRevID = nullslice; - Retained doc = _db->useCollection(collection)->getDocument(request->docID, true, kDocGetUpgraded); + Retained doc = _db->useCollection(collection)->getDocument(request->docID, true, kDocGetAll); if ( doc ) { if ( doc->selectRevision(request->revID, true) ) root = doc->getProperties(); if ( root ) request->flags = doc->selectedRev().flags; diff --git a/Replicator/Pusher.cc b/Replicator/Pusher.cc index 3a9f98ef0..842dfb892 100644 --- a/Replicator/Pusher.cc +++ b/Replicator/Pusher.cc @@ -414,7 +414,7 @@ namespace litecore::repl { bool Pusher::shouldRetryConflictWithNewerAncestor(RevToSend* rev, slice receivedRevID) { if ( !_proposeChanges ) return false; try { - Retained doc = _db->getDoc(getCollection(), rev->docID, kDocGetUpgraded); + Retained doc = _db->getDoc(getCollection(), rev->docID, kDocGetAll); if ( doc && C4Document::equalRevIDs(doc->revID(), rev->revID) ) { if ( receivedRevID && receivedRevID != rev->remoteAncestorRevID ) { // Remote ancestor received in proposeChanges response, so try with From 6b624ca1b72d672c7b65defaffc9417051701524 Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Wed, 11 Sep 2024 14:26:47 -0700 Subject: [PATCH 4/4] Fixed some broken test code c4doc_getRevisionHistory requires doc be loaded with kDocGetAll. VectorDoc correctly threw an exception on this. Also, all this is doing is checking the current rev, so no need to get the history. --- Replicator/tests/ReplicatorCollectionTest.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Replicator/tests/ReplicatorCollectionTest.cc b/Replicator/tests/ReplicatorCollectionTest.cc index 0dd6f88cc..bbe3f21b4 100644 --- a/Replicator/tests/ReplicatorCollectionTest.cc +++ b/Replicator/tests/ReplicatorCollectionTest.cc @@ -579,8 +579,7 @@ TEST_CASE_METHOD(ReplicatorCollectionTest, "Multiple Collections Incremental Rev for ( const auto& coll_doc : docsWithIncrementalRevisions ) { c4::ref doc = c4coll_getDoc(coll_doc.first, coll_doc.second, true, kDocGetMetadata, ERROR_INFO()); CHECK(doc); - alloc_slice hist = c4doc_getRevisionHistory(doc, 1, nullptr, 0); - if ( isRevTrees() ) CHECK(3 == c4rev_getGeneration(hist)); + if ( doc && isRevTrees() ) CHECK(c4rev_getGeneration(doc->revID) == 3); } }