From c0e0b631c8bb246cacca3e1f9204224867029f6b Mon Sep 17 00:00:00 2001 From: Doug Hoyte Date: Thu, 12 Sep 2024 16:52:11 -0400 Subject: [PATCH] update for new DB format --- src/apps/web/AlgoParser.h | 10 ++++---- src/apps/web/AlgoScanner.h | 48 +++++++++++++++++++++----------------- src/apps/web/WebData.h | 17 +++++++++----- src/apps/web/WebWriter.cpp | 18 +++++++------- 4 files changed, 53 insertions(+), 40 deletions(-) diff --git a/src/apps/web/AlgoParser.h b/src/apps/web/AlgoParser.h index 9ccd20eb..8511743c 100644 --- a/src/apps/web/AlgoParser.h +++ b/src/apps/web/AlgoParser.h @@ -157,11 +157,13 @@ ::string_view v){ if (parsedKey.s == pubkey && parsedKey.n1 == kind) { auto levId = lmdb::from_sv(v); auto ev = lookupEventByLevId(txn, levId); + PackedEventView packed(ev.buf); - for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) { - if ((char)tagPair->key() != 'p') continue; - output.insert(std::string(sv(tagPair->val()))); - } + packed.foreachTag([&](char tagName, std::string_view tagVal){ + if (tagName != 'p') return true; + output.insert(std::string(tagVal)); + return true; + }); } return false; diff --git a/src/apps/web/AlgoScanner.h b/src/apps/web/AlgoScanner.h index 6ec4583c..b0da9f8e 100644 --- a/src/apps/web/AlgoScanner.h +++ b/src/apps/web/AlgoScanner.h @@ -36,21 +36,24 @@ struct AlgoScanner { if (output.size() > limit) return false; auto ev = lookupEventByLevId(txn, lmdb::from_sv(v)); - auto kind = ev.flat_nested()->kind(); - auto id = sv(ev.flat_nested()->id()); + PackedEventView packed(ev.buf); + + auto kind = packed.kind(); + auto id = packed.id(); if (kind == 1) { - auto pubkey = std::string(sv(ev.flat_nested()->pubkey())); + auto pubkey = std::string(packed.pubkey()); bool foundETag = false; - for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) { - if ((char)tagPair->key() == 'e') { - auto tagEventId = std::string(sv(tagPair->val())); + packed.foreachTag([&](char tagName, std::string_view tagVal){ + if (tagName == 'e') { + auto tagEventId = std::string(tagVal); eventInfoCache.emplace(tagEventId, EventInfo{}); eventInfoCache[tagEventId].comments++; foundETag = true; } - } + return true; + }); if (foundETag) return true; // not root event eventInfoCache.emplace(id, EventInfo{}); @@ -62,18 +65,19 @@ struct AlgoScanner { output.emplace_back(FilteredEvent{ev.primaryKeyId, std::string(id), eventInfo}); } else if (kind == 7) { - auto pubkey = std::string(sv(ev.flat_nested()->pubkey())); + auto pubkey = std::string(packed.pubkey()); //if (a.voters && !a.voters->contains(pubkey)) return true; - const auto &tagsArr = *(ev.flat_nested()->tagsFixed32()); - for (auto it = tagsArr.rbegin(); it != tagsArr.rend(); ++it) { - auto tagPair = *it; - if ((char)tagPair->key() == 'e') { - auto tagEventId = std::string(sv(tagPair->val())); - eventInfoCache.emplace(tagEventId, EventInfo{}); - eventInfoCache[tagEventId].score++; - break; - } + std::optional lastETag; + packed.foreachTag([&](char tagName, std::string_view tagVal){ + if (tagName == 'e') lastETag = tagVal; + return true; + }); + + if (lastETag) { + auto tagEventId = std::string(*lastETag); + eventInfoCache.emplace(tagEventId, EventInfo{}); + eventInfoCache[tagEventId].score++; } } @@ -97,11 +101,13 @@ ::string_view v){ if (parsedKey.s == pubkey && parsedKey.n1 == kind) { auto levId = lmdb::from_sv(v); auto ev = lookupEventByLevId(txn, levId); + PackedEventView packed(ev.buf); - for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) { - if ((char)tagPair->key() != 'p') continue; - output.insert(std::string(sv(tagPair->val()))); - } + packed.foreachTag([&](char tagName, std::string_view tagVal){ + if (tagName != 'p') return true; + output.insert(std::string(tagVal)); + return true; + }); } return false; diff --git a/src/apps/web/WebData.h b/src/apps/web/WebData.h index 22758370..93f83bcc 100644 --- a/src/apps/web/WebData.h +++ b/src/apps/web/WebData.h @@ -142,9 +142,10 @@ struct User { auto levId = lmdb::from_sv(v); auto ev = lookupEventByLevId(txn, levId); + PackedEventView packed(ev.buf); - if (ev.flat_nested()->kind() == 3) { - auto pubkey = std::string(sv(ev.flat_nested()->pubkey())); + if (packed.kind() == 3) { + auto pubkey = std::string(packed.pubkey()); if (!alreadySeen.contains(pubkey)) { alreadySeen.insert(pubkey); @@ -206,19 +207,23 @@ struct Event { std::string getId() const { - return std::string(sv(ev.flat_nested()->id())); + PackedEventView packed(ev.buf); + return std::string(packed.id()); } uint64_t getKind() const { - return ev.flat_nested()->kind(); + PackedEventView packed(ev.buf); + return packed.kind(); } uint64_t getCreatedAt() const { - return ev.flat_nested()->created_at(); + PackedEventView packed(ev.buf); + return packed.created_at(); } std::string getPubkey() const { - return std::string(sv(ev.flat_nested()->pubkey())); + PackedEventView packed(ev.buf); + return std::string(packed.pubkey()); } std::string getNoteId() const { diff --git a/src/apps/web/WebWriter.cpp b/src/apps/web/WebWriter.cpp index b310a9c1..7d1ac08b 100644 --- a/src/apps/web/WebWriter.cpp +++ b/src/apps/web/WebWriter.cpp @@ -10,16 +10,16 @@ void WebServer::runWriter(ThreadPool::Thread &thr) { secp256k1_context *secpCtx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); PluginEventSifter writePolicy; + NegentropyFilterCache neFilterCache; + while(1) { auto newMsgs = thr.inbox.pop_all(); - auto now = hoytech::curr_time_us(); std::vector newEvents; for (auto &newMsg : newMsgs) { if (auto msg = std::get_if(&newMsg.msg)) { auto &req = msg->req; - EventSourceType sourceType = req.ipAddr.size() == 4 ? EventSourceType::IP4 : EventSourceType::IP6; Url u(req.url); if (u.path.size() != 1 || u.path[0] != "submit-post") { @@ -27,23 +27,23 @@ void WebServer::runWriter(ThreadPool::Thread &thr) { continue; } - std::string flatStr, jsonStr; + std::string packedStr, jsonStr; try { tao::json::value json = tao::json::from_string(req.body); - parseAndVerifyEvent(json, secpCtx, true, true, flatStr, jsonStr); + parseAndVerifyEvent(json, secpCtx, true, true, packedStr, jsonStr); } catch(std::exception &e) { sendHttpResponse(req, tao::json::to_string(tao::json::value({{ "message", e.what() }})), "404 Not Found", "application/json; charset=utf-8"); continue; } - newEvents.emplace_back(std::move(flatStr), std::move(jsonStr), now, sourceType, req.ipAddr, &req); + newEvents.emplace_back(std::move(packedStr), std::move(jsonStr), &req); } } try { auto txn = env.txn_rw(); - writeEvents(txn, newEvents); + writeEvents(txn, neFilterCache, newEvents); txn.commit(); } catch (std::exception &e) { LE << "Error writing " << newEvents.size() << " events: " << e.what(); @@ -61,8 +61,8 @@ void WebServer::runWriter(ThreadPool::Thread &thr) { for (auto &newEvent : newEvents) { - auto *flat = flatbuffers::GetRoot(newEvent.flatStr.data()); - auto eventIdHex = to_hex(sv(flat->id())); + PackedEventView packed(newEvent.packedStr); + auto eventIdHex = to_hex(packed.id()); tao::json::value output = tao::json::empty_object; std::string message; @@ -71,7 +71,7 @@ void WebServer::runWriter(ThreadPool::Thread &thr) { LI << "Inserted event. id=" << eventIdHex << " levId=" << newEvent.levId; output["message"] = message = "ok"; output["written"] = true; - output["event"] = encodeBech32Simple("note", sv(flat->id())); + output["event"] = encodeBech32Simple("note", packed.id()); } else if (newEvent.status == EventWriteStatus::Duplicate) { output["message"] = message = "duplicate: have this event"; output["written"] = true;