Skip to content

Commit

Permalink
update for new DB format
Browse files Browse the repository at this point in the history
  • Loading branch information
@hoytech
hoytech committed Sep 12, 2024
1 parent 62f361f commit c0e0b63
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 40 deletions.
10 changes: 6 additions & 4 deletions src/apps/web/AlgoParser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,13 @@ ::string_view v){
if (parsedKey.s == pubkey && parsedKey.n1 == kind) {
auto levId = lmdb::from_sv<uint64_t>(v);
auto ev = lookupEventByLevId(txn, levId);
PackedEventView packed(ev.buf);

for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) {
if ((char)tagPair->key() != 'p') continue;
output.insert(std::string(sv(tagPair->val())));
}
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName != 'p') return true;
output.insert(std::string(tagVal));
return true;
});
}

return false;
Expand Down
48 changes: 27 additions & 21 deletions src/apps/web/AlgoScanner.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,21 +36,24 @@ struct AlgoScanner {
if (output.size() > limit) return false;

auto ev = lookupEventByLevId(txn, lmdb::from_sv<uint64_t>(v));
auto kind = ev.flat_nested()->kind();
auto id = sv(ev.flat_nested()->id());
PackedEventView packed(ev.buf);

auto kind = packed.kind();
auto id = packed.id();

if (kind == 1) {
auto pubkey = std::string(sv(ev.flat_nested()->pubkey()));
auto pubkey = std::string(packed.pubkey());

bool foundETag = false;
for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) {
if ((char)tagPair->key() == 'e') {
auto tagEventId = std::string(sv(tagPair->val()));
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName == 'e') {
auto tagEventId = std::string(tagVal);
eventInfoCache.emplace(tagEventId, EventInfo{});
eventInfoCache[tagEventId].comments++;
foundETag = true;
}
}
return true;
});
if (foundETag) return true; // not root event

eventInfoCache.emplace(id, EventInfo{});
Expand All @@ -62,18 +65,19 @@ struct AlgoScanner {

output.emplace_back(FilteredEvent{ev.primaryKeyId, std::string(id), eventInfo});
} else if (kind == 7) {
auto pubkey = std::string(sv(ev.flat_nested()->pubkey()));
auto pubkey = std::string(packed.pubkey());
//if (a.voters && !a.voters->contains(pubkey)) return true;

const auto &tagsArr = *(ev.flat_nested()->tagsFixed32());
for (auto it = tagsArr.rbegin(); it != tagsArr.rend(); ++it) {
auto tagPair = *it;
if ((char)tagPair->key() == 'e') {
auto tagEventId = std::string(sv(tagPair->val()));
eventInfoCache.emplace(tagEventId, EventInfo{});
eventInfoCache[tagEventId].score++;
break;
}
std::optional<std::string_view> lastETag;
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName == 'e') lastETag = tagVal;
return true;
});

if (lastETag) {
auto tagEventId = std::string(*lastETag);
eventInfoCache.emplace(tagEventId, EventInfo{});
eventInfoCache[tagEventId].score++;
}
}

Expand All @@ -97,11 +101,13 @@ ::string_view v){
if (parsedKey.s == pubkey && parsedKey.n1 == kind) {
auto levId = lmdb::from_sv<uint64_t>(v);
auto ev = lookupEventByLevId(txn, levId);
PackedEventView packed(ev.buf);

for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) {
if ((char)tagPair->key() != 'p') continue;
output.insert(std::string(sv(tagPair->val())));
}
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName != 'p') return true;
output.insert(std::string(tagVal));
return true;
});
}

return false;
Expand Down
17 changes: 11 additions & 6 deletions src/apps/web/WebData.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,10 @@ struct User {

auto levId = lmdb::from_sv<uint64_t>(v);
auto ev = lookupEventByLevId(txn, levId);
PackedEventView packed(ev.buf);

if (ev.flat_nested()->kind() == 3) {
auto pubkey = std::string(sv(ev.flat_nested()->pubkey()));
if (packed.kind() == 3) {
auto pubkey = std::string(packed.pubkey());

if (!alreadySeen.contains(pubkey)) {
alreadySeen.insert(pubkey);
Expand Down Expand Up @@ -206,19 +207,23 @@ struct Event {


std::string getId() const {
return std::string(sv(ev.flat_nested()->id()));
PackedEventView packed(ev.buf);
return std::string(packed.id());
}

uint64_t getKind() const {
return ev.flat_nested()->kind();
PackedEventView packed(ev.buf);
return packed.kind();
}

uint64_t getCreatedAt() const {
return ev.flat_nested()->created_at();
PackedEventView packed(ev.buf);
return packed.created_at();
}

std::string getPubkey() const {
return std::string(sv(ev.flat_nested()->pubkey()));
PackedEventView packed(ev.buf);
return std::string(packed.pubkey());
}

std::string getNoteId() const {
Expand Down
18 changes: 9 additions & 9 deletions src/apps/web/WebWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,40 +10,40 @@ void WebServer::runWriter(ThreadPool<MsgWebWriter>::Thread &thr) {
secp256k1_context *secpCtx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY);
PluginEventSifter writePolicy;

NegentropyFilterCache neFilterCache;

while(1) {
auto newMsgs = thr.inbox.pop_all();
auto now = hoytech::curr_time_us();

std::vector<EventToWrite> newEvents;

for (auto &newMsg : newMsgs) {
if (auto msg = std::get_if<MsgWebWriter::Request>(&newMsg.msg)) {
auto &req = msg->req;
EventSourceType sourceType = req.ipAddr.size() == 4 ? EventSourceType::IP4 : EventSourceType::IP6;

Url u(req.url);
if (u.path.size() != 1 || u.path[0] != "submit-post") {
sendHttpResponse(req, "Not found", "404 Not Found");
continue;
}

std::string flatStr, jsonStr;
std::string packedStr, jsonStr;

try {
tao::json::value json = tao::json::from_string(req.body);
parseAndVerifyEvent(json, secpCtx, true, true, flatStr, jsonStr);
parseAndVerifyEvent(json, secpCtx, true, true, packedStr, jsonStr);
} catch(std::exception &e) {
sendHttpResponse(req, tao::json::to_string(tao::json::value({{ "message", e.what() }})), "404 Not Found", "application/json; charset=utf-8");
continue;
}

newEvents.emplace_back(std::move(flatStr), std::move(jsonStr), now, sourceType, req.ipAddr, &req);
newEvents.emplace_back(std::move(packedStr), std::move(jsonStr), &req);
}
}

try {
auto txn = env.txn_rw();
writeEvents(txn, newEvents);
writeEvents(txn, neFilterCache, newEvents);
txn.commit();
} catch (std::exception &e) {
LE << "Error writing " << newEvents.size() << " events: " << e.what();
Expand All @@ -61,8 +61,8 @@ void WebServer::runWriter(ThreadPool<MsgWebWriter>::Thread &thr) {


for (auto &newEvent : newEvents) {
auto *flat = flatbuffers::GetRoot<NostrIndex::Event>(newEvent.flatStr.data());
auto eventIdHex = to_hex(sv(flat->id()));
PackedEventView packed(newEvent.packedStr);
auto eventIdHex = to_hex(packed.id());

tao::json::value output = tao::json::empty_object;
std::string message;
Expand All @@ -71,7 +71,7 @@ void WebServer::runWriter(ThreadPool<MsgWebWriter>::Thread &thr) {
LI << "Inserted event. id=" << eventIdHex << " levId=" << newEvent.levId;
output["message"] = message = "ok";
output["written"] = true;
output["event"] = encodeBech32Simple("note", sv(flat->id()));
output["event"] = encodeBech32Simple("note", packed.id());
} else if (newEvent.status == EventWriteStatus::Duplicate) {
output["message"] = message = "duplicate: have this event";
output["written"] = true;
Expand Down

0 comments on commit c0e0b63

Please sign in to comment.