From b7b86386646a60a8cf5328b3c8f5989d832a4560 Mon Sep 17 00:00:00 2001 From: Dmytro Kozhevin Date: Mon, 23 May 2022 17:25:24 -0400 Subject: [PATCH] Added utility to dump ledger state from buckets to JSON for debugging (optionally using `jq` for post-processing). The full ledger in this format currently takes ~19GB and is not really feasible for meaningful processing. Hence initially I introduce the following ways to reduce the output (all of which can be combined): - Filter expressions for conditionally selection the entries - Limit to only recently updated entries - Limit the output entry count For the sake of filtering I introduced XDR field extraction utility, that can be used for other dynamic operations on XDR, e.g. group-by and/or reduction. However, for now I'm keeping this minimalistic until some concrete use-cases that aren't feasible with the current approach appear. The current approach is a simple iteration over buckets from newer to older. This probably could be optimized (e.g. with parallel execution or XDR of intermediate results), but for now the performance seems acceptable: ~80s on my laptop for filtering the full ledger with a reasonably small output set. --- .gitignore | 3 + Builds/VisualStudio/stellar-core.vcxproj | 34 +- .../VisualStudio/stellar-core.vcxproj.filters | 42 ++ docs/software/commands.md | 28 +- src/Makefile.am | 15 + src/bucket/BucketManager.h | 23 + src/bucket/BucketManagerImpl.cpp | 110 +++++ src/bucket/BucketManagerImpl.h | 5 + src/main/ApplicationUtils.cpp | 62 +++ src/main/ApplicationUtils.h | 4 + src/main/CommandLine.cpp | 57 ++- src/util/xdrquery/XDRFieldResolver.h | 417 +++++++++++++++++ src/util/xdrquery/XDRMatcher.cpp | 32 ++ src/util/xdrquery/XDRMatcher.h | 47 ++ src/util/xdrquery/XDRQueryError.h | 21 + src/util/xdrquery/XDRQueryEval.cpp | 307 ++++++++++++ src/util/xdrquery/XDRQueryEval.h | 160 +++++++ src/util/xdrquery/XDRQueryParser.yy | 153 ++++++ src/util/xdrquery/XDRQueryScanner.ll | 68 +++ src/util/xdrquery/tests/XDRQueryTests.cpp | 443 ++++++++++++++++++ 20 files changed, 2022 insertions(+), 9 deletions(-) create mode 100644 src/util/xdrquery/XDRFieldResolver.h create mode 100644 src/util/xdrquery/XDRMatcher.cpp create mode 100644 src/util/xdrquery/XDRMatcher.h create mode 100644 src/util/xdrquery/XDRQueryError.h create mode 100644 src/util/xdrquery/XDRQueryEval.cpp create mode 100644 src/util/xdrquery/XDRQueryEval.h create mode 100644 src/util/xdrquery/XDRQueryParser.yy create mode 100644 src/util/xdrquery/XDRQueryScanner.ll create mode 100644 src/util/xdrquery/tests/XDRQueryTests.cpp diff --git a/.gitignore b/.gitignore index 42961a599c..fb9640e673 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,6 @@ min-testcases/ *.vcxproj.user **/.vs/* +/src/util/xdrquery/XDRQueryScanner.cpp +/src/util/xdrquery/XDRQueryParser.h +/src/util/xdrquery/XDRQueryParser.cpp diff --git a/Builds/VisualStudio/stellar-core.vcxproj b/Builds/VisualStudio/stellar-core.vcxproj index 01399cd719..0fd0cbd06b 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj +++ b/Builds/VisualStudio/stellar-core.vcxproj @@ -130,7 +130,7 @@ false ProgramDatabase - /bigobj %(AdditionalOptions) + /bigobj /Zc:__cplusplus %(AdditionalOptions) stdcpp17 @@ -188,7 +188,7 @@ exit /b 0 false ProgramDatabase - /bigobj %(AdditionalOptions) + /bigobj /Zc:__cplusplus %(AdditionalOptions) stdcpp17 @@ -247,7 +247,7 @@ exit /b 0 false ProgramDatabase - /bigobj %(AdditionalOptions) + /bigobj /Zc:__cplusplus %(AdditionalOptions) stdcpp17 @@ -305,7 +305,7 @@ exit /b 0 true 4060;4100;4127;4324;4408;4510;4512;4582;4583;4592 false - /bigobj %(AdditionalOptions) + /bigobj /Zc:__cplusplus %(AdditionalOptions) stdcpp17 false @@ -363,7 +363,7 @@ exit /b 0 true 4060;4100;4127;4324;4408;4510;4512;4582;4583;4592 false - /bigobj %(AdditionalOptions) + /bigobj /Zc:__cplusplus %(AdditionalOptions) stdcpp17 false @@ -541,6 +541,11 @@ exit /b 0 + + + + + @@ -937,6 +942,11 @@ exit /b 0 + + + + + @@ -1442,6 +1452,20 @@ $(OutDir)\bin\cxxbridge.exe ..\..\src\rust\src\lib.rs --output src\$(Configurati + + Document + bison --defines=../../src/util/xdrquery/XDRQueryParser.h --output=../../src/util/xdrquery/XDRQueryParser.cpp ../../src/util/xdrquery/XDRQueryParser.yy + running bison for XDRQueryParser.yy + ../../src/util/xdrquery/XDRQueryParser.h + true + + + Document + flex --outfile=../../src/util/xdrquery/XDRQueryScanner.cpp ../../src/util/xdrquery/XDRQueryScanner.ll + running flex for XDRQueryScanner.ll + ../../src/util/xdrquery/XDRQueryScanner.cpp + true + diff --git a/Builds/VisualStudio/stellar-core.vcxproj.filters b/Builds/VisualStudio/stellar-core.vcxproj.filters index 09a75f0e6c..40c6fdcb6f 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj.filters +++ b/Builds/VisualStudio/stellar-core.vcxproj.filters @@ -172,6 +172,12 @@ {57d45a48-6030-4dff-96ce-fd587f319529} + + {166a403b-4f13-4505-9953-97ea92e5d855} + + + {60ef25c4-d908-4795-83cf-a348673fe4c4} + @@ -1248,6 +1254,21 @@ herder + + util\xdrquery + + + util\xdrquery + + + util\xdrquery + + + util\xdrquery + + + util\xdrquery\tests + @@ -2177,6 +2198,21 @@ xdr\generated + + util\xdrquery + + + util\xdrquery + + + util\xdrquery + + + util\xdrquery + + + util\xdrquery + @@ -2368,6 +2404,12 @@ xdr\curr + + util\xdrquery + + + util\xdrquery + diff --git a/docs/software/commands.md b/docs/software/commands.md index 57aea49ea2..046e4176d2 100644 --- a/docs/software/commands.md +++ b/docs/software/commands.md @@ -34,7 +34,33 @@ Command options can only by placed after command. private key. For example: `$ stellar-core convert-id SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2` - +* **dump-ledger**: Dumps the current ledger state from bucket files into + JSON **--output-file** with optional filtering. **--last-ledgers** option + allows to only dump the ledger entries that were last modified within that + many ledgers. **--limit** option limits the output to that many arbitrary + records. **--filter-query** allows to specify a filtering expression over + `LedgerEntry` XDR. Expression should evaluate to boolean and consist of + field paths, comparisons, literals, boolean operators (`&&`, ` ||`) and + parentheses. The field values are consistent with `print-xdr` JSON + representation: enums are represented as their name strings, account ids as + encoded strings, hashes as hex strings etc. Filtering is useful to minimize + the output JSON size and then optionally process it further with tools like + `jq`. Query examples: + + * `data.type == 'OFFER'` - dump only offers + * `data.account.accountID == 'GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' || + data.trustLine.accountID == "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"` + - dump only account and trustline entries for the specified account. + * `data.account.inflationDest != NULL` - dump accounts that have an optional + `inflationDest` field set. + * `data.offer.selling.assetCode == 'FOOBAR' && + data.offer.selling.issuer == 'GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'` - + dump offers that are selling the specified asset. + * `data.trustLine.ext.v1.liabilities.buying < data.trustLine.ext.v1.liabilities.selling` - + dump trustlines that have buying liabilites less than selling liabilites + * `(data.account.balance < 100000000 || data.account.balance >= 2000000000) + && data.account.numSubEntries > 2` - dump accounts with certain balance + and sub entries count, demonstrates more complex expression * **dump-xdr **: Dumps the given XDR file and then exits. * **encode-asset --code --issuer **: Prints a base-64 encoded asset. Prints the native asset if neither `code` nor `issuer` is given. diff --git a/src/Makefile.am b/src/Makefile.am index 3e070a95aa..c343158d7b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -32,6 +32,21 @@ SUFFIXES = .x .h .rs .x.h: $(XDRC) -hh -pedantic -o $@ $< +BISON=bison +FLEX=flex + +$(srcdir)/util/xdrquery/XDRQueryScanner.cpp: $(srcdir)/util/xdrquery/XDRQueryScanner.ll + $(FLEX) --outfile=$@ $< + +$(srcdir)/util/xdrquery/XDRQueryParser.cpp: $(srcdir)/util/xdrquery/XDRQueryParser.yy + $(BISON) --defines=$(srcdir)/util/xdrquery/XDRQueryParser.h --output=$@ $< + +$(srcdir)/util/xdrquery/XDRQueryParser.h: $(srcdir)/util/xdrquery/XDRQueryParser.cpp + touch $@ + +BUILT_SOURCES += $(srcdir)/util/xdrquery/XDRQueryScanner.cpp $(srcdir)/util/xdrquery/XDRQueryParser.h $(srcdir)/util/xdrquery/XDRQueryParser.cpp +stellar_core_SOURCES += $(srcdir)/util/xdrquery/XDRQueryScanner.cpp $(srcdir)/util/xdrquery/XDRQueryParser.h $(srcdir)/util/xdrquery/XDRQueryParser.cpp + # Old automakes have buggy dependency tracking for conditional generated # sources. We work around this here by making rust/RustBridge.{cpp,h} generated # in all cases, and just empty in the non-rust case. Also because of the way old diff --git a/src/bucket/BucketManager.h b/src/bucket/BucketManager.h index 66e8f135e5..7705f6b4c4 100644 --- a/src/bucket/BucketManager.h +++ b/src/bucket/BucketManager.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "medida/timer_context.h" @@ -230,6 +231,28 @@ class BucketManager : NonMovableOrCopyable virtual std::shared_ptr mergeBuckets(HistoryArchiveState const& has) = 0; + // Visits all the active ledger entries or subset thereof. + // + // The order in which the entries are visited is not defined, but roughly + // goes from more fresh entries to the older ones. + // + // This accepts two visitors. `filterEntry` has to return `true` + // if the ledger entry can *potentially* be accepted. The passed entry isn't + // necessarily fresh or even alive. `acceptEntry` will only get the fresh + // alive entries that have passed the filter. If it returns `false` the + // iteration will immediately finish. + // + // When `minLedger` is specified, only entries that have been modified at + // `minLedger` or later are visited. + // + // When `filterEntry` and `acceptEntry` always return `true`, this is + // equivalent to iterating over `loadCompleteLedgerState`, so the same + // memory/runtime implications apply. + virtual void visitLedgerEntries( + HistoryArchiveState const& has, std::optional minLedger, + std::function const& filterEntry, + std::function const& acceptEntry) = 0; + // Schedule a Work class that verifies the hashes of all referenced buckets // on background threads. virtual std::shared_ptr diff --git a/src/bucket/BucketManagerImpl.cpp b/src/bucket/BucketManagerImpl.cpp index 985cfec604..f2f1a9a4fb 100644 --- a/src/bucket/BucketManagerImpl.cpp +++ b/src/bucket/BucketManagerImpl.cpp @@ -966,6 +966,116 @@ BucketManagerImpl::mergeBuckets(HistoryArchiveState const& has) return out.getBucket(*this); } +static bool +visitEntriesInBucket(std::shared_ptr b, std::string const& name, + std::optional minLedger, + std::function const& filterEntry, + std::function const& acceptEntry, + UnorderedSet& processedEntries) +{ + using namespace std::chrono; + medida::Timer timer; + + UnorderedMap bucketEntries; + bool stopIteration = false; + timer.Time([&]() { + for (BucketInputIterator in(b); in; ++in) + { + BucketEntry const& e = *in; + if (e.type() == LIVEENTRY || e.type() == INITENTRY) + { + if (minLedger && + e.liveEntry().lastModifiedLedgerSeq < *minLedger) + { + stopIteration = true; + continue; + } + if (!filterEntry(e.liveEntry())) + { + continue; + } + auto key = LedgerEntryKey(e.liveEntry()); + if (processedEntries.find(key) != processedEntries.end()) + { + continue; + } + bucketEntries[key] = e.liveEntry(); + } + else + { + if (e.type() != DEADENTRY) + { + std::string err = "Malformed bucket: unexpected " + "non-INIT/LIVE/DEAD entry."; + CLOG_ERROR(Bucket, "{}", err); + throw std::runtime_error(err); + } + bucketEntries.erase(e.deadEntry()); + processedEntries.insert(e.deadEntry()); + } + } + for (auto const& [key, entry] : bucketEntries) + { + processedEntries.insert(key); + if (!acceptEntry(entry)) + { + stopIteration = true; + break; + } + } + }); + nanoseconds ns = + timer.duration_unit() * static_cast(timer.max()); + milliseconds ms = duration_cast(ns); + size_t bytesPerSec = (b->getSize() * 1000 / (1 + ms.count())); + CLOG_INFO(Bucket, "Processed {}-byte bucket file '{}' in {} ({}/s)", + b->getSize(), name, ms, formatSize(bytesPerSec)); + return !stopIteration; +} + +void +BucketManagerImpl::visitLedgerEntries( + HistoryArchiveState const& has, std::optional minLedger, + std::function const& filterEntry, + std::function const& acceptEntry) +{ + UnorderedSet deletedEntries; + std::vector> hashes; + for (uint32_t i = 0; i < BucketList::kNumLevels; ++i) + { + HistoryStateBucket const& hsb = has.currentBuckets.at(i); + hashes.emplace_back(hexToBin256(hsb.curr), + fmt::format(FMT_STRING("curr {:d}"), i)); + hashes.emplace_back(hexToBin256(hsb.snap), + fmt::format(FMT_STRING("snap {:d}"), i)); + } + medida::Timer timer; + timer.Time([&]() { + for (auto const& pair : hashes) + { + if (isZero(pair.first)) + { + continue; + } + auto b = getBucketByHash(pair.first); + if (!b) + { + throw std::runtime_error(std::string("missing bucket: ") + + binToHex(pair.first)); + } + if (!visitEntriesInBucket(b, pair.second, minLedger, filterEntry, + acceptEntry, deletedEntries)) + { + break; + } + } + }); + auto ns = timer.duration_unit() * + static_cast(timer.max()); + CLOG_INFO(Bucket, "Total ledger processing time: {}", + std::chrono::duration_cast(ns)); +} + std::shared_ptr BucketManagerImpl::scheduleVerifyReferencedBucketsWork() { diff --git a/src/bucket/BucketManagerImpl.h b/src/bucket/BucketManagerImpl.h index 29fea66a94..95941b53b9 100644 --- a/src/bucket/BucketManagerImpl.h +++ b/src/bucket/BucketManagerImpl.h @@ -143,6 +143,11 @@ class BucketManagerImpl : public BucketManager std::shared_ptr mergeBuckets(HistoryArchiveState const& has) override; + void visitLedgerEntries( + HistoryArchiveState const& has, std::optional minLedger, + std::function const& filterEntry, + std::function const& acceptEntry) override; + std::shared_ptr scheduleVerifyReferencedBucketsWork() override; }; diff --git a/src/main/ApplicationUtils.cpp b/src/main/ApplicationUtils.cpp index 5da2eb2e0a..60cd28224b 100644 --- a/src/main/ApplicationUtils.cpp +++ b/src/main/ApplicationUtils.cpp @@ -25,6 +25,8 @@ #include "overlay/OverlayManager.h" #include "util/GlobalChecks.h" #include "util/Logging.h" +#include "util/XDRCereal.h" +#include "util/xdrquery/XDRMatcher.h" #include "work/WorkScheduler.h" #include @@ -436,6 +438,66 @@ mergeBucketList(Config cfg, std::string const& outputDir) } } +int +dumpLedger(Config cfg, std::string const& outputFile, + std::optional filterQuery, + std::optional lastModifiedLedgerCount, + std::optional limit) +{ + VirtualClock clock; + cfg.setNoListen(); + Application::pointer app = Application::create(clock, cfg, false); + app->getLedgerManager().loadLastKnownLedger(nullptr); + auto& lm = app->getLedgerManager(); + HistoryArchiveState has = lm.getLastClosedLedgerHAS(); + std::optional minLedger; + if (lastModifiedLedgerCount) + { + uint32_t lclNum = lm.getLastClosedLedgerNum(); + if (lclNum >= *lastModifiedLedgerCount) + { + minLedger = lclNum - *lastModifiedLedgerCount; + } + else + { + minLedger = 0; + } + } + std::optional matcher; + if (filterQuery) + { + matcher.emplace(*filterQuery); + } + std::ofstream ofs(outputFile); + ofs << "{\"entries\": ["; + + auto& bm = app->getBucketManager(); + uint64_t entryCount = 0; + try + { + bm.visitLedgerEntries( + has, minLedger, + [&](LedgerEntry const& entry) { + return !matcher || matcher->matchXDR(entry); + }, + [&](LedgerEntry const& entry) { + if (entryCount != 0) + { + ofs << "," << std::endl; + } + ofs << xdr_to_string(entry, "entry", true); + ++entryCount; + return !limit || entryCount < *limit; + }); + } + catch (xdrquery::XDRQueryError& e) + { + LOG_ERROR(DEFAULT_LOG, "Filter query error: {}", e.what()); + } + ofs << "]}"; + return 0; +} + void setForceSCPFlag() { diff --git a/src/main/ApplicationUtils.h b/src/main/ApplicationUtils.h index f709ad4705..744b4c7a29 100644 --- a/src/main/ApplicationUtils.h +++ b/src/main/ApplicationUtils.h @@ -23,6 +23,10 @@ void initializeDatabase(Config cfg); void httpCommand(std::string const& command, unsigned short port); int selfCheck(Config cfg); int mergeBucketList(Config cfg, std::string const& outputDir); +int dumpLedger(Config cfg, std::string const& outputFile, + std::optional filterQuery, + std::optional lastModifiedLedgerCount, + std::optional limit); void showOfflineInfo(Config cfg); int reportLastHistoryCheckpoint(Config cfg, std::string const& outputFile); #ifdef BUILD_TESTS diff --git a/src/main/CommandLine.cpp b/src/main/CommandLine.cpp index d7332b1562..33327c2a47 100644 --- a/src/main/CommandLine.cpp +++ b/src/main/CommandLine.cpp @@ -357,21 +357,49 @@ inMemoryParser(bool& inMemory) { return clara::Opt{inMemory}["--in-memory"]( "store working ledger in memory rather than database"); -}; +} clara::Opt startAtLedgerParser(uint32_t& startAtLedger) { return clara::Opt{startAtLedger, "LEDGER"}["--start-at-ledger"]( "start in-memory run with replay from historical ledger number"); -}; +} clara::Opt startAtHashParser(std::string& startAtHash) { return clara::Opt{startAtHash, "HASH"}["--start-at-hash"]( "start in-memory run with replay from historical ledger hash"); -}; +} + +clara::Opt +filterQueryParser(std::optional& filterQuery) +{ + return clara::Opt{[&](std::string const& arg) { filterQuery = arg; }, + "FILTER-QUERY"}["--filter-query"]( + "query to filter ledger entries"); +} + +clara::Opt +lastModifiedLedgerCountParser( + std::optional& lastModifiedLedgerCount) +{ + return clara::Opt{[&](std::string const& arg) { + lastModifiedLedgerCount = std::stoul(arg); + }, + "LAST-LEDGERS"}["--last-ledgers"]( + "filter out ledger entries that were modified more than this many " + "ledgers ago"); +} + +clara::Opt +limitParser(std::optional& limit) +{ + return clara::Opt{[&](std::string const& arg) { limit = std::stoull(arg); }, + "LIMIT"}["--limit"]( + "limit the output to this many ledger entries"); +} int runWithHelp(CommandLineArgs const& args, @@ -1033,6 +1061,27 @@ runMergeBucketList(CommandLineArgs const& args) [&] { return mergeBucketList(configOption.getConfig(), outputDir); }); } +int +runDumpLedger(CommandLineArgs const& args) +{ + CommandLine::ConfigOption configOption; + std::string outputFile; + std::optional filterQuery; + std::optional lastModifiedLedgerCount; + std::optional limit; + return runWithHelp(args, + {configurationParser(configOption), + outputFileParser(outputFile).required(), + filterQueryParser(filterQuery), + lastModifiedLedgerCountParser(lastModifiedLedgerCount), + limitParser(limit)}, + [&] { + return dumpLedger(configOption.getConfig(), + outputFile, filterQuery, + lastModifiedLedgerCount, limit); + }); +} + int runNewDB(CommandLineArgs const& args) { @@ -1631,6 +1680,8 @@ handleCommandLine(int argc, char* const* argv) {"convert-id", "displays ID in all known forms", runConvertId}, {"diag-bucket-stats", "reports statistics on the content of a bucket", diagBucketStats}, + {"dump-ledger", "dumps the current ledger state as JSON for debugging", + runDumpLedger}, {"dump-xdr", "dump an XDR file, for debugging", runDumpXDR}, {"encode-asset", "Print an encoded asset in base 64 for debugging", runEncodeAsset}, diff --git a/src/util/xdrquery/XDRFieldResolver.h b/src/util/xdrquery/XDRFieldResolver.h new file mode 100644 index 0000000000..3d71588f3f --- /dev/null +++ b/src/util/xdrquery/XDRFieldResolver.h @@ -0,0 +1,417 @@ +// Copyright 2022 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#pragma once + +#include +#include +#include + +#include "crypto/Hex.h" +#include "crypto/KeyUtils.h" +#include "crypto/SecretKey.h" +#include "util/GlobalChecks.h" +#include "util/types.h" +#include "util/xdrquery/XDRQueryError.h" +#include "util/xdrquery/XDRQueryEval.h" +#include "xdr/Stellar-ledger-entries.h" +#include "xdrpp/marshal.h" +#include "xdrpp/types.h" + +namespace xdrquery +{ +namespace internal +{ +using namespace xdr; +using namespace stellar; + +struct XDRFieldResolver +{ + XDRFieldResolver(std::vector const& fieldPath, bool validate) + : mFieldPath(fieldPath) + , mPathIter(mFieldPath.cbegin()) + , mValidate(validate) + { + } + + ResultType const& + getResult() + { + releaseAssert(!mValidate); + return mResult; + } + + bool + isValid() const + { + releaseAssert(mValidate); + return mPathIter == mFieldPath.end(); + } + + template + typename std::enable_if_t::is_numeric && + !xdr_traits::is_enum> + operator()(T const& t, char const* fieldName) + { + if (checkLeafField(fieldName)) + { + mResult = t; + } + } + + // Retrieve enums as their XDR string representation. + template + typename std::enable_if_t::is_enum> + operator()(T const& t, char const* fieldName) + { + if (checkLeafField(fieldName)) + { + mResult = std::string(xdr_traits::enum_name(t)); + } + } + + // Retrieve public keys in standard string representation. + template + typename std::enable_if_t> + operator()(T const& k, char const* fieldName) + { + if (checkLeafField(fieldName)) + { + mResult = stellar::KeyUtils::toStrKey(k); + } + } + + template + typename std::enable_if_t || + std::is_same_v> + operator()(T const& asset, char const* fieldName) + { + if (!matchFieldToPath(fieldName)) + { + return; + } + ++mPathIter; + switch (asset.type()) + { + case ASSET_TYPE_NATIVE: + if (mPathIter == mFieldPath.end()) + { + // If non-leaf field is requested, then we must be looking for + // non-native asset. + mResult.emplace().emplace() = "NATIVE"; + } + break; + case ASSET_TYPE_POOL_SHARE: + processPoolAsset(asset); + break; + case ASSET_TYPE_CREDIT_ALPHANUM4: + case ASSET_TYPE_CREDIT_ALPHANUM12: + { + std::string code; + if (asset.type() == ASSET_TYPE_CREDIT_ALPHANUM4) + { + stellar::assetCodeToStr(asset.alphaNum4().assetCode, code); + } + else + { + stellar::assetCodeToStr(asset.alphaNum12().assetCode, code); + } + + processString(code, "assetCode"); + (*this)(stellar::getIssuer(asset), "issuer"); + break; + } + default: + mResult = "UNKNOWN"; + break; + } + + if (mValidate) + { + validateAsset(asset, fieldName); + } + } + + template + void + operator()(xstring const& t, char const* fieldName) + { + if (checkLeafField(fieldName)) + { + mResult = std::string(t); + } + } + + template + void + operator()(xdr::opaque_vec const& v, char const* fieldName) + { + if (checkLeafField(fieldName)) + { + mResult = binToHex(ByteSlice(v.data(), v.size())); + } + } + + template + void + operator()(xdr::opaque_array const& v, char const* fieldName) + { + if (checkLeafField(fieldName)) + { + mResult = binToHex(ByteSlice(v.data(), v.size())); + } + } + + template + void + operator()(xdr::pointer const& ptr, char const* fieldName) + { + if (ptr) + { + archive(*this, *ptr, fieldName); + } + else + { + if (checkLeafField(fieldName)) + { + mResult = NullField(); + return; + } + if (mValidate && matchFieldToPath(fieldName)) + { + ++mPathIter; + // Create an instance of the field for validation. + T t; + xdr_traits::save(*this, t); + } + } + } + + template + typename std::enable_if_t::is_container> + operator()(T const& t, char const* fieldName) + { + if (matchFieldToPath(fieldName)) + { + throw XDRQueryError( + fmt::format(FMT_STRING("Array fields are not supported: '{}'."), + fieldName)); + } + } + + template + typename std::enable_if_t< + xdr_traits::is_union && !std::is_same_v && + !std::is_same_v && !std::is_same_v && + !xdr_traits::is_container> + operator()(T const& t, char const* fieldName) + { + if (!matchFieldToPath(fieldName)) + { + // Archive is first called with an empty 'virtual' XDR field + // representing the whole struct. + if (fieldName == nullptr && mPathIter == mFieldPath.begin()) + { + xdr_traits::save(*this, t); + } + return; + } + if (++mPathIter == mFieldPath.end()) + { + throw XDRQueryError("Field path must end with a primitive field."); + } + xdr_traits::save(*this, t); + if (mValidate) + { + validateUnion(t, fieldName); + } + } + + template + typename std::enable_if_t< + xdr_traits::is_class && !std::is_same_v && + !std::is_same_v && !std::is_same_v && + !xdr_traits::is_union && !xdr_traits::is_container> + operator()(T const& t, char const* fieldName) + { + if (!matchFieldToPath(fieldName)) + { + // Archive is first called with an empty 'virtual' XDR field + // representing the whole struct. + if (fieldName == nullptr && mPathIter == mFieldPath.begin()) + { + xdr_traits::save(*this, t); + } + return; + } + if (++mPathIter == mFieldPath.end()) + { + throw XDRQueryError("Field path must end with a primitive field."); + } + xdr_traits::save(*this, t); + } + + private: + bool + matchFieldToPath(char const* fieldName) const + { + return fieldName != nullptr && mPathIter != mFieldPath.end() && + *mPathIter == fieldName; + } + + bool + checkLeafField(char const* fieldName) + { + if (!matchFieldToPath(fieldName)) + { + return false; + } + if (++mPathIter != mFieldPath.end()) + { + throw XDRQueryError( + fmt::format(FMT_STRING("Encountered leaf field in the middle " + "of the field path: '{}'."), + fieldName)); + } + return true; + } + + bool + checkMaybeLeafField(char const* fieldName, bool& isLeaf) + { + if (!matchFieldToPath(fieldName)) + { + return false; + } + return ++mPathIter != mFieldPath.end(); + } + + void + processString(std::string const& s, char const* fieldName) + { + if (checkLeafField(fieldName)) + { + mResult = s; + } + } + + void + processPoolAsset(Asset const& asset) + { + throw std::runtime_error("Unexpected asset type for the pool asset."); + } + + void + processPoolAsset(TrustLineAsset const& asset) + { + (*this)(asset.liquidityPoolID(), "liquidityPoolID"); + } + + template + typename std::enable_if_t::is_union> + validateUnion(T const& t, char const* fieldName) + { + // The field could have been already matched if it was XDR discriminant. + if (mPathIter == mFieldPath.end()) + { + return; + } + for (auto const c : t._xdr_case_values()) + { + auto unionFieldName = xdr_traits::union_field_name(c); + if (unionFieldName == nullptr || unionFieldName != *mPathIter) + { + continue; + } + auto tCopy = t; + tCopy._xdr_discriminant(c, false); + tCopy._xdr_with_mem_ptr(field_archiver, c, *this, tCopy, + unionFieldName); + break; + } + } + + template + typename std::enable_if_t || + std::is_same_v> + validateAsset(T const& asset, char const* fieldName) + { + // The field could have been already matched if it was a native asset. + if (mPathIter == mFieldPath.end()) + { + return; + } + + // Check if the field is requested from liquidity pool share. + if constexpr (std::is_same_v) + { + checkLeafField("liquidityPoolID"); + } + // For regular asset we allow 'assetCode' and 'issuer' fields. + checkLeafField("assetCode"); + checkLeafField("issuer"); + } + + std::vector const& mFieldPath; + std::vector::const_iterator mPathIter; + ResultType mResult; + bool mValidate = false; +}; + +} // namespace internal + +// Returns the value of the field in `xdrMessage` specified by `fieldPath`. +// When path goes through a union option that is not selected, returns +// std::nullopt. +// When path ends with an optional field and the field is not set, +// returns `NullField`. +// Some types have special overrides, mostly consistent with XDR-to-JSON +// representation: +// - enums are represented by strings +// - `AccountID` has a standard string representation ('GXYZ...') +// - Assets are simplified to {`assetCode`, `issuer`} struct (and +// `liquidityPoolID` for the pool shares) +// - Fixed size byte arrays are represented by the hex strings +template +ResultType +getXDRField(T const& xdrMessage, std::vector const& fieldPath) +{ + internal::XDRFieldResolver resolver(fieldPath, false); + xdr::xdr_argpack_archive(resolver, xdrMessage); + return resolver.getResult(); +} + +// Like `getXDRField`, but throws XDRQueryError when path is not present in XDR +// message (accounting for all the union variants and optional fields). +template +ResultType +getXDRFieldValidated(T const& xdrMessage, + std::vector const& fieldPath) +{ + internal::XDRFieldResolver validator(fieldPath, true); + xdr::xdr_argpack_archive(validator, xdrMessage); + if (!validator.isValid()) + { + throw XDRQueryError(fmt::format(FMT_STRING("Invalid field path: '{}'."), + fmt::join(fieldPath, "."))); + } + return getXDRField(xdrMessage, fieldPath); +} + +} // namespace xdrquery + +namespace xdr +{ +template <> struct archive_adapter +{ + template + static void + apply(xdrquery::internal::XDRFieldResolver& ar, T&& t, + char const* fieldName) + { + ar(std::forward(t), fieldName); + } +}; + +} diff --git a/src/util/xdrquery/XDRMatcher.cpp b/src/util/xdrquery/XDRMatcher.cpp new file mode 100644 index 0000000000..babd160fc7 --- /dev/null +++ b/src/util/xdrquery/XDRMatcher.cpp @@ -0,0 +1,32 @@ +// Copyright 2022 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "util/xdrquery/XDRMatcher.h" + +#include "util/xdrquery/XDRQueryParser.h" + +#include +#include +#include + +namespace xdrquery +{ +XDRMatcher::XDRMatcher(std::string const& query) : mQuery(query) +{ +} + +bool +XDRMatcher::matchInternal(FieldResolver const& fieldResolver) +{ + // Lazily parse the query in order to simplify exception handling as we + // might throw XDRQueryError both during query parsing and query execution + // against XDR. + if (mEvalRoot == nullptr) + { + mEvalRoot = parseXDRQuery(mQuery); + } + return mEvalRoot->evalBool(fieldResolver); +} + +} // namespace xdrquery diff --git a/src/util/xdrquery/XDRMatcher.h b/src/util/xdrquery/XDRMatcher.h new file mode 100644 index 0000000000..b2c490f8e1 --- /dev/null +++ b/src/util/xdrquery/XDRMatcher.h @@ -0,0 +1,47 @@ +// Copyright 2022 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#pragma once + +#include "util/xdrquery/XDRFieldResolver.h" +#include "util/xdrquery/XDRQueryEval.h" +#include +#include + +namespace xdrquery +{ +// Helper to match multiple XDR messages of the same type using the provided +// query. +// Queries may consist of literals, XDR fields, comparisons and boolean +// operations, e.g. +// `data.account.balance >= 100000 || data.trustLine.balance < 5000` +// See more examples in `XDRQueryTests`. +class XDRMatcher +{ + public: + XDRMatcher(std::string const& query); + + template + bool + matchXDR(T const& xdrMessage) + { + return matchInternal( + [&xdrMessage, this](std::vector const& fieldPath) { + if (mFirstMatch) + { + mFirstMatch = false; + return getXDRFieldValidated(xdrMessage, fieldPath); + } + return getXDRField(xdrMessage, fieldPath); + }); + } + + private: + bool matchInternal(FieldResolver const& fieldResolver); + + std::string const mQuery; + std::unique_ptr mEvalRoot; + bool mFirstMatch = true; +}; +} // namespace xdrquery diff --git a/src/util/xdrquery/XDRQueryError.h b/src/util/xdrquery/XDRQueryError.h new file mode 100644 index 0000000000..cca60c5f64 --- /dev/null +++ b/src/util/xdrquery/XDRQueryError.h @@ -0,0 +1,21 @@ +// Copyright 2022 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#pragma once + +#include + +namespace xdrquery +{ +// A common exception for any XDR query-related errors, including parsing, field +// resolution, type checks etc. +class XDRQueryError : public std::invalid_argument +{ + public: + explicit XDRQueryError(std::string const& msg) : std::invalid_argument{msg} + { + } + virtual ~XDRQueryError() = default; +}; +} // namespace xdrquery diff --git a/src/util/xdrquery/XDRQueryEval.cpp b/src/util/xdrquery/XDRQueryEval.cpp new file mode 100644 index 0000000000..c715dee84c --- /dev/null +++ b/src/util/xdrquery/XDRQueryEval.cpp @@ -0,0 +1,307 @@ +// Copyright 2022 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#include "util/xdrquery/XDRQueryEval.h" +#include "fmt/format.h" +#include "util/xdrquery/XDRQueryError.h" + +namespace fmt +{ +template <> struct formatter +{ + template + auto + format(xdrquery::NullField, FormatContext& ctx) + { + return format_to(ctx.out(), "NULL"); + } +}; +} // fmt + +namespace xdrquery +{ +bool +NullField::operator==(NullField other) const +{ + return operationNotSupported(); +} +bool +NullField::operator!=(NullField other) const +{ + return operationNotSupported(); +} +bool +NullField::operator<(NullField other) const +{ + return operationNotSupported(); +} +bool +NullField::operator<=(NullField other) const +{ + return operationNotSupported(); +} +bool +NullField::operator>(NullField other) const +{ + return operationNotSupported(); +} +bool +NullField::operator>=(NullField other) const +{ + return operationNotSupported(); +} +bool +NullField::operationNotSupported() const +{ + throw std::runtime_error("Null fields should not be compared directly."); + return false; +} + +LiteralNode::LiteralNode(LiteralNodeType valueType, std::string const& val) + : mType(valueType), mValue(val) +{ + if (mType == LiteralNodeType::NULL_LITERAL) + { + mValue = NullField(); + } +} + +ResultType +LiteralNode::eval(FieldResolver const& fieldResolver) const +{ + return mValue; +} + +EvalNodeType +LiteralNode::getType() const +{ + return EvalNodeType::LITERAL; +} + +void +LiteralNode::resolveIntType(ResultValueType const& fieldValue, + std::vector const& fieldPath) const +{ + if (std::holds_alternative(fieldValue)) + { + std::string valueStr = + std::visit([](auto&& v) { return fmt::to_string(v); }, *mValue); + throw XDRQueryError(fmt::format( + FMT_STRING("String field '{}' is compared with int value: {}."), + fmt::join(fieldPath, "."), valueStr)); + } + std::string valueStr = std::get(*mValue); + try + { + mValue = std::visit( + [&valueStr](auto&& v) -> ResultType { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + auto v = std::stoi(valueStr); + if (v > std::numeric_limits::max()) + { + throw std::out_of_range(""); + } + return std::make_optional( + std::in_place_type, std::stoi(valueStr)); + } + else if constexpr (std::is_same_v) + return std::make_optional( + std::in_place_type, std::stoll(valueStr)); + else if constexpr (std::is_same_v) + { + auto v = std::stoul(valueStr); + if (v > std::numeric_limits::max()) + { + throw std::out_of_range(""); + } + return std::make_optional( + std::in_place_type, v); + } + else if constexpr (std::is_same_v) + return std::make_optional( + std::in_place_type, std::stoull(valueStr)); + throw std::runtime_error("Unexpected field type."); + }, + fieldValue); + } + catch (std::out_of_range&) + { + throw XDRQueryError(fmt::format( + FMT_STRING("Value for field '{}' is out of type range: {}."), + fmt::join(fieldPath, "."), valueStr)); + } +} + +FieldNode::FieldNode(std::string const& initField) +{ + mFieldPath.push_back(initField); +} + +ResultType +FieldNode::eval(FieldResolver const& fieldResolver) const +{ + return fieldResolver(mFieldPath); +} + +EvalNodeType +FieldNode::getType() const +{ + return EvalNodeType::FIELD; +} + +ResultType +BoolEvalNode::eval(FieldResolver const& fieldResolver) const +{ + return evalBool(fieldResolver); +} + +BoolOpNode::BoolOpNode(BoolOpNodeType nodeType, + std::unique_ptr left, + std::unique_ptr right) + : mType(nodeType), mLeft(std::move(left)), mRight(std::move(right)) +{ +} + +bool +BoolOpNode::evalBool(FieldResolver const& fieldResolver) const +{ + switch (mType) + { + case BoolOpNodeType::AND: + return mLeft->evalBool(fieldResolver) && + mRight->evalBool(fieldResolver); + case BoolOpNodeType::OR: + return mLeft->evalBool(fieldResolver) || + mRight->evalBool(fieldResolver); + } +} + +EvalNodeType +BoolOpNode::getType() const +{ + return EvalNodeType(); +} + +ComparisonNode::ComparisonNode(ComparisonNodeType nodeType, + std::unique_ptr left, + std::unique_ptr right) + : mType(nodeType), mLeft(std::move(left)), mRight(std::move(right)) +{ + // Keep the field as the left argument for simplicity of type check during + // evaluation. + if (mRight->getType() == EvalNodeType::FIELD) + { + std::swap(mLeft, mRight); + // Invert the operation as we have swapped operands. + switch (mType) + { + case ComparisonNodeType::LT: + mType = ComparisonNodeType::GT; + break; + case ComparisonNodeType::LE: + mType = ComparisonNodeType::GE; + break; + case ComparisonNodeType::GT: + mType = ComparisonNodeType::LT; + break; + case ComparisonNodeType::GE: + mType = ComparisonNodeType::LE; + break; + default: + break; + } + } +} + +bool +ComparisonNode::evalBool(FieldResolver const& fieldResolver) const +{ + auto leftType = mLeft->getType(); + auto leftVal = mLeft->eval(fieldResolver); + + if (!leftVal) + { + return false; + } + + auto rightType = mRight->getType(); + if (leftType == EvalNodeType::FIELD && rightType == EvalNodeType::LITERAL) + { + // Lazily resolve the type of the int literal using the field type. + // This allows to correctly check the literal range and simplifies the + // comparisons. + auto* lit = static_cast(mRight.get()); + if (lit->mType == LiteralNodeType::INT && + std::holds_alternative(*lit->mValue)) + { + auto* field = static_cast(mLeft.get()); + lit->resolveIntType(*leftVal, field->mFieldPath); + } + } + auto rightVal = mRight->eval(fieldResolver); + if (!rightVal) + { + return false; + } + + bool leftIsNull = std::holds_alternative(*leftVal); + bool rightIsNull = std::holds_alternative(*rightVal); + if (leftIsNull || rightIsNull) + { + return compareNullFields(leftIsNull, rightIsNull); + } + + if (leftVal->index() != rightVal->index()) + { + auto valueToStr = [](auto&& v) { return fmt::to_string(v); }; + throw XDRQueryError(fmt::format( + FMT_STRING("Type mismatch between values `{}` and `{}`."), + std::visit(valueToStr, *leftVal), + std::visit(valueToStr, *rightVal))); + } + + switch (mType) + { + case ComparisonNodeType::EQ: + return *leftVal == *rightVal; + case ComparisonNodeType::NE: + return *leftVal != *rightVal; + case ComparisonNodeType::LT: + return *leftVal < *rightVal; + case ComparisonNodeType::LE: + return *leftVal <= *rightVal; + case ComparisonNodeType::GT: + return *leftVal > *rightVal; + case ComparisonNodeType::GE: + return *leftVal >= *rightVal; + } +} + +EvalNodeType +ComparisonNode::getType() const +{ + return EvalNodeType::COMPARISON_OP; +} + +bool +ComparisonNode::compareNullFields(bool leftIsNull, bool rightIsNull) const +{ + switch (mType) + { + case ComparisonNodeType::EQ: + return leftIsNull == rightIsNull; + case ComparisonNodeType::NE: + return leftIsNull != rightIsNull; + case ComparisonNodeType::LT: + case ComparisonNodeType::LE: + case ComparisonNodeType::GT: + case ComparisonNodeType::GE: + throw XDRQueryError( + "Fields can only be compared with `NULL` using `==` and `!=`."); + } +} + +} // namespace xdrquery diff --git a/src/util/xdrquery/XDRQueryEval.h b/src/util/xdrquery/XDRQueryEval.h new file mode 100644 index 0000000000..c2e8d88bc6 --- /dev/null +++ b/src/util/xdrquery/XDRQueryEval.h @@ -0,0 +1,160 @@ +// Copyright 2022 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// This is a simple engine for evaluating boolean expresions containing literals +// and XDR fields. +namespace xdrquery +{ +// This type represents an optional XDR fields that is not set. +struct NullField +{ + bool operator==(NullField other) const; + bool operator!=(NullField other) const; + bool operator<(NullField other) const; + bool operator<=(NullField other) const; + bool operator>(NullField other) const; + bool operator>=(NullField other) const; + + private: + bool operationNotSupported() const; +}; + +// All the possible intermediate expression result types that are defined +// values. +using ResultValueType = std::variant; + +// All the possible expression result types. When this is set to std::nullopt, +// this represents an expression that cannot be meaningfully evaluated to either +// `true` or `false`. Currently that's only set to std::nullopt for the cases +// when an XDR union has an alternative selected that is not in the field path. +using ResultType = std::optional; + +// A function that resolves the field path to an actual value. +using FieldResolver = + std::function const&)>; + +enum class EvalNodeType +{ + LITERAL, + FIELD, + BOOL_OP, + COMPARISON_OP +}; + +// Expression node that can be evaluated. +struct EvalNode +{ + virtual ResultType eval(FieldResolver const& fieldResolver) const = 0; + virtual EvalNodeType getType() const = 0; + + virtual ~EvalNode() = default; +}; + +enum class LiteralNodeType +{ + INT, + STR, + NULL_LITERAL +}; + +// Node representing literal in the expression. +struct LiteralNode : public EvalNode +{ + LiteralNode(LiteralNodeType valueType, std::string const& val); + + ResultType eval(FieldResolver const& fieldResolver) const override; + + EvalNodeType getType() const override; + + // We only resolve integer literals when they're compared to XDR fields and + // for simplicity do that lazily via calling this function in eval(). Hence + // it has to be `const` and `mValue` has to be mutable. + void resolveIntType(ResultValueType const& fieldValue, + std::vector const& fieldPath) const; + + LiteralNodeType mType; + mutable ResultType mValue; +}; + +// Node representing an XDR field in expression. +struct FieldNode : public EvalNode +{ + FieldNode(std::string const& initField); + + ResultType eval(FieldResolver const& fieldResolver) const override; + + EvalNodeType getType() const override; + + std::vector mFieldPath; +}; + +// `EvalNode` that always has a `bool` evaluation result. +struct BoolEvalNode : public EvalNode +{ + ResultType eval(FieldResolver const& fieldResolver) const override; + + virtual bool evalBool(FieldResolver const& fieldResolver) const = 0; +}; + +enum class BoolOpNodeType +{ + AND, + OR +}; + +// Node for binary bool operations. +struct BoolOpNode : public BoolEvalNode +{ + BoolOpNode(BoolOpNodeType nodeType, std::unique_ptr left, + std::unique_ptr right); + + bool evalBool(FieldResolver const& fieldResolver) const override; + + EvalNodeType getType() const override; + + private: + BoolOpNodeType mType; + std::unique_ptr mLeft; + std::unique_ptr mRight; +}; + +enum class ComparisonNodeType +{ + EQ, + NE, + LT, + LE, + GT, + GE +}; + +// Node for comparing arbitrary values. Values have to have the same type. +struct ComparisonNode : public BoolEvalNode +{ + ComparisonNode(ComparisonNodeType nodeType, std::unique_ptr left, + std::unique_ptr right); + + bool evalBool(FieldResolver const& fieldResolver) const override; + + EvalNodeType getType() const override; + + private: + bool compareNullFields(bool leftIsNull, bool rightIsNull) const; + + ComparisonNodeType mType; + std::unique_ptr mLeft; + std::unique_ptr mRight; +}; +} // namespace xdrquery diff --git a/src/util/xdrquery/XDRQueryParser.yy b/src/util/xdrquery/XDRQueryParser.yy new file mode 100644 index 0000000000..e25d5dcabc --- /dev/null +++ b/src/util/xdrquery/XDRQueryParser.yy @@ -0,0 +1,153 @@ +/* Copyright 2022 Stellar Development Foundation and contributors. Licensed + under the Apache License, Version 2.0. See the COPYING file at the root + of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 */ +%skeleton "lalr1.cc" /* -*- C++ -*- */ +%require "3.2" + +%code requires +{ +#include "util/xdrquery/XDRQueryError.h" +#include "util/xdrquery/XDRQueryEval.h" + +#include +} + +%code provides +{ +#define YY_DECL xdrquery::XDRQueryParser::symbol_type yylex() +YY_DECL; + +namespace xdrquery +{ +std::unique_ptr +parseXDRQuery(std::string const& query); +} // namespace xdrquery +} + +%define api.value.type variant +%define api.parser.class { XDRQueryParser } +%define api.namespace { xdrquery } +%define api.token.prefix {TOKEN_} +%define api.token.constructor + +%parse-param { std::unique_ptr& root } + +%token ID +%token INT +%token STR + +%token NULL + +%token AND "&&" +%token OR "||" + +%token EQ "==" +%token NE "!=" +%token GT ">" +%token GE ">=" +%token LT "<" +%token LE "<=" + +%token LPAREN "(" +%token RPAREN ")" + +%token DOT "." + +%left "||" +%left "&&" +%left "==" "!=" ">" ">=" "<" "<=" + +%type > literal operand +%type > comparison_expr logic_expr +%type > field + +%% + +statement: logic_expr { root = std::move($1); } + +logic_expr: comparison_expr { $$ = std::move($1); } + | "(" logic_expr ")" { $$ = std::move($2); } + | logic_expr "&&" logic_expr { + $$ = std::make_unique(BoolOpNodeType::AND, + std::move($1), std::move($3)); } + | logic_expr "||" logic_expr { + $$ = std::make_unique(BoolOpNodeType::OR, + std::move($1), std::move($3)); } + +comparison_expr: operand "==" operand { + $$ = std::make_unique(ComparisonNodeType::EQ, + std::move($1), std::move($3)); } + | operand "!=" operand { + $$ = std::make_unique(ComparisonNodeType::NE, + std::move($1), std::move($3)); } + | operand "<" operand { + $$ = std::make_unique(ComparisonNodeType::LT, + std::move($1), std::move($3)); } + | operand "<=" operand { + $$ = std::make_unique(ComparisonNodeType::LE, + std::move($1), std::move($3)); } + | operand ">" operand { + $$ = std::make_unique(ComparisonNodeType::GT, + std::move($1), std::move($3)); } + | operand ">=" operand { + $$ = std::make_unique(ComparisonNodeType::GE, + std::move($1), std::move($3)); } + +operand: literal { $$ = std::move($1); } + | field { $$ = std::move($1); } + +literal: INT { $$ = std::make_unique(LiteralNodeType::INT, $1); } + | STR { $$ = std::make_unique(LiteralNodeType::STR, $1); } + | NULL { $$ = std::make_unique(LiteralNodeType::NULL_LITERAL, ""); } + +field: ID { $$ = std::make_unique($1); } + | field "." ID { $1->mFieldPath.push_back($3); $$ = std::move($1); } + +%% + +#ifdef __has_feature + #if __has_feature(address_sanitizer) + #define ASAN_ENABLED + #endif +#else + #ifdef __SANITIZE_ADDRESS__ + #define ASAN_ENABLED + #endif +#endif + +#ifdef ASAN_ENABLED +#include +#endif + +void beginScan(char const* s); +void endScan(); + +namespace xdrquery +{ +void +XDRQueryParser::error(std::string const& error) +{ + throw XDRQueryError("Parsing error: '" + error + "'."); +} + +std::unique_ptr +parseXDRQuery(std::string const& query) +{ + // LeakSantizer (likely) incorrectly identifies some small leaks in + // lexer, hence disable it for the query parsing. According + // to the docs, calling `yylex_destoy` should be enough to do the proper + // cleanup. +#ifdef ASAN_ENABLED + __lsan_disable(); +#endif + beginScan(query.c_str()); + std::unique_ptr root; + XDRQueryParser parser(root); + parser(); + endScan(); +#ifdef ASAN_ENABLED + __lsan_enable(); +#endif + return root; +} +} // namespace xdrquery diff --git a/src/util/xdrquery/XDRQueryScanner.ll b/src/util/xdrquery/XDRQueryScanner.ll new file mode 100644 index 0000000000..5f581bd166 --- /dev/null +++ b/src/util/xdrquery/XDRQueryScanner.ll @@ -0,0 +1,68 @@ +/* Copyright 2022 Stellar Development Foundation and contributors. Licensed + under the Apache License, Version 2.0. See the COPYING file at the root + of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 */ +%{ +#ifdef _MSC_VER +#include +#define popen _popen +#define pclose _pclose +#define access _access +#define isatty _isatty +#define fileno _fileno +#else +#include +#endif + +#include "util/xdrquery/XDRQueryParser.h" +%} + +%option noyywrap +%option nounput noinput +%option batch + +IDENTIFIER [a-zA-Z_][a-zA-Z_0-9]* +INT -?[0-9]+ +STRING (\"[^"\n]*\")|('[^'\n]*') +WHITESPACE [ \t\r\n] + +%% + +NULL { return xdrquery::XDRQueryParser::make_NULL(); } + +{IDENTIFIER} { return xdrquery::XDRQueryParser::make_ID(yytext); } +{INT} { return xdrquery::XDRQueryParser::make_INT(yytext); } + +"&&" { return xdrquery::XDRQueryParser::make_AND(); } +"||" { return xdrquery::XDRQueryParser::make_OR(); } + +"==" { return xdrquery::XDRQueryParser::make_EQ(); } +"!=" { return xdrquery::XDRQueryParser::make_NE(); } +">=" { return xdrquery::XDRQueryParser::make_GE(); } +">" { return xdrquery::XDRQueryParser::make_GT(); } +"<=" { return xdrquery::XDRQueryParser::make_LE(); } +"<" { return xdrquery::XDRQueryParser::make_LT(); } + +"(" { return xdrquery::XDRQueryParser::make_LPAREN(); } +")" { return xdrquery::XDRQueryParser::make_RPAREN(); } + +"." { return xdrquery::XDRQueryParser::make_DOT(); } + +{STRING} { + std::string s(yytext + 1); + s.pop_back(); + return xdrquery::XDRQueryParser::make_STR(s); +} + +{WHITESPACE}+ /* discard */; + +. { throw xdrquery::XDRQueryParser::syntax_error("Unexpected character: " + std::string(yytext)); } + +%% + +void beginScan(char const* s) { + yy_scan_string(s); +} + +void endScan() { + yylex_destroy(); +} diff --git a/src/util/xdrquery/tests/XDRQueryTests.cpp b/src/util/xdrquery/tests/XDRQueryTests.cpp new file mode 100644 index 0000000000..1366694207 --- /dev/null +++ b/src/util/xdrquery/tests/XDRQueryTests.cpp @@ -0,0 +1,443 @@ +// Copyright 2022 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 +#include "util/types.h" +#include "util/xdrquery/XDRFieldResolver.h" +#include "util/xdrquery/XDRMatcher.h" +#include "xdr/Stellar-ledger-entries.h" + +#include +#include + +namespace xdrquery +{ +namespace +{ +using namespace stellar; + +LedgerEntry +makeAccountEntry(int64_t balance) +{ + LedgerEntry accountEntry; + accountEntry.data.type(ACCOUNT); + auto& account = accountEntry.data.account(); + account.accountID.ed25519().back() = 111; + account.balance = balance; + account.seqNum = std::numeric_limits::max(); + account.numSubEntries = std::numeric_limits::min(); + account.inflationDest.activate().ed25519()[0] = 78; + account.homeDomain = "home_domain"; + account.thresholds[0] = 1; + account.thresholds[2] = 2; + account.ext.v(1); + account.ext.v1().liabilities.buying = std::numeric_limits::min(); + account.ext.v1().ext.v(2); + account.ext.v1().ext.v2().ext.v(3); + account.ext.v1().ext.v2().ext.v3().seqTime = + std::numeric_limits::max(); + return accountEntry; +} + +LedgerEntry +makeOfferEntry(std::string const& assetName) +{ + LedgerEntry offerEntry; + offerEntry.data.type(OFFER); + if (assetName.length() <= 4) + { + offerEntry.data.offer().selling.type(ASSET_TYPE_CREDIT_ALPHANUM4); + strToAssetCode(offerEntry.data.offer().selling.alphaNum4().assetCode, + assetName); + offerEntry.data.offer().selling.alphaNum4().issuer.ed25519().back() = + 111; + } + else + { + offerEntry.data.offer().selling.type(ASSET_TYPE_CREDIT_ALPHANUM12); + strToAssetCode(offerEntry.data.offer().selling.alphaNum12().assetCode, + assetName); + offerEntry.data.offer().selling.alphaNum12().issuer.ed25519().back() = + 111; + } + + return offerEntry; +} + +TEST_CASE("XDR field resolver", "[xdrquery]") +{ + auto accountEntry = makeAccountEntry(123); + auto const& account = accountEntry.data.account(); + + SECTION("int32 field") + { + Price price; + price.n = std::numeric_limits::min(); + price.d = std::numeric_limits::max(); + SECTION("negative") + { + auto field = getXDRFieldValidated(price, {"n"}); + REQUIRE(std::get(*field) == price.n); + } + SECTION("positive") + { + auto field = getXDRFieldValidated(price, {"d"}); + REQUIRE(std::get(*field) == price.d); + } + } + + SECTION("uint32 field") + { + auto field = getXDRFieldValidated(accountEntry, + {"data", "account", "numSubEntries"}); + REQUIRE(std::get(*field) == account.numSubEntries); + } + + SECTION("int64 field") + { + SECTION("negative") + { + auto field = getXDRFieldValidated( + accountEntry, + {"data", "account", "ext", "v1", "liabilities", "buying"}); + REQUIRE(std::get(*field) == + account.ext.v1().liabilities.buying); + } + SECTION("positive") + { + auto field = getXDRFieldValidated(accountEntry, + {"data", "account", "seqNum"}); + REQUIRE(std::get(*field) == account.seqNum); + } + } + + SECTION("uint64 field") + { + auto field = getXDRFieldValidated( + accountEntry, {"data", "account", "ext", "v1", "ext", "v2", "ext", + "v3", "seqTime"}); + REQUIRE(std::get(*field) == + account.ext.v1().ext.v2().ext.v3().seqTime); + } + + SECTION("string field") + { + auto field = getXDRFieldValidated(accountEntry, + {"data", "account", "homeDomain"}); + REQUIRE(std::get(*field) == account.homeDomain); + } + + SECTION("bytes field") + { + auto field = getXDRFieldValidated(accountEntry, + {"data", "account", "thresholds"}); + REQUIRE(std::get(*field) == "01000200"); + } + + SECTION("enum field") + { + auto field = getXDRFieldValidated(accountEntry, {"data", "type"}); + REQUIRE(std::get(*field) == "ACCOUNT"); + } + + SECTION("null field") + { + LedgerEntry e; + e.data.type(ACCOUNT); + auto field = + getXDRFieldValidated(e, {"data", "account", "inflationDest"}); + REQUIRE(std::holds_alternative(*field)); + } + + SECTION("public key field") + { + SECTION("non-optional") + { + auto field = getXDRFieldValidated(accountEntry, + {"data", "account", "accountID"}); + REQUIRE(std::get(*field) == + "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG6ELY"); + } + SECTION("optional") + { + auto field = getXDRFieldValidated( + accountEntry, {"data", "account", "inflationDest"}); + REQUIRE(std::get(*field) == + "GBHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB2HL"); + } + } + + SECTION("asset field") + { + auto testAsset = [&](auto& entry, auto& asset, + std::vector const& fieldPath) { + SECTION("native") + { + asset.type(ASSET_TYPE_NATIVE); + + auto field = getXDRFieldValidated(entry, fieldPath); + REQUIRE(std::get(*field) == "NATIVE"); + } + auto testAlphaNum = [&](auto& alphaNum, std::string const& code) { + strToAssetCode(alphaNum.assetCode, code); + std::copy(account.accountID.ed25519().begin(), + account.accountID.ed25519().end(), + alphaNum.issuer.ed25519().begin()); + SECTION("assetCode") + { + auto currFieldPath = fieldPath; + currFieldPath.push_back("assetCode"); + auto field = getXDRFieldValidated(entry, currFieldPath); + REQUIRE(std::get(*field) == code); + } + + SECTION("issuer") + { + auto currFieldPath = fieldPath; + currFieldPath.push_back("issuer"); + auto field = getXDRFieldValidated(entry, currFieldPath); + REQUIRE(std::get(*field) == + "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AG6ELY"); + } + }; + SECTION("alphanum4") + { + asset.type(ASSET_TYPE_CREDIT_ALPHANUM4); + testAlphaNum(asset.alphaNum4(), "USD"); + } + SECTION("alphanum12") + { + asset.type(ASSET_TYPE_CREDIT_ALPHANUM12); + testAlphaNum(asset.alphaNum12(), "USD123"); + } + }; + SECTION("regular asset") + { + OfferEntry entry; + testAsset(entry, entry.selling, {"selling"}); + } + SECTION("trustline asset") + { + TrustLineEntry entry; + testAsset(entry, entry.asset, {"asset"}); + + SECTION("pool share") + { + entry.asset.type(ASSET_TYPE_POOL_SHARE); + entry.asset.liquidityPoolID()[0] = 1; + entry.asset.liquidityPoolID()[2] = 2; + auto field = + getXDRFieldValidated(entry, {"asset", "liquidityPoolID"}); + REQUIRE(std::get(*field) == + "010002000000000000000000000000000000000000000000000000" + "0000000000"); + } + } + } + + SECTION("non-matching union returns nullopt") + { + auto field = getXDRFieldValidated(accountEntry, + {"data", "trustLine", "accountID"}); + REQUIRE(!field); + } + + SECTION("bad paths throw exception") + { + SECTION("incorrect leaf name") + { + REQUIRE_THROWS_AS( + getXDRFieldValidated(accountEntry, + {"data", "account", "noSuchField"}), + XDRQueryError); + } + SECTION("incorrect struct field name") + { + REQUIRE_THROWS_AS( + getXDRFieldValidated(accountEntry, + {"data2", "account", "balance"}), + XDRQueryError); + } + SECTION("incorrect union field name") + { + REQUIRE_THROWS_AS( + getXDRFieldValidated(accountEntry, + {"data", "account2", "balance"}), + XDRQueryError); + } + + SECTION("leaf field in the middle") + { + REQUIRE_THROWS_AS( + getXDRFieldValidated( + accountEntry, {"data", "account", "balance", "balance2"}), + XDRQueryError); + } + SECTION("non-leaf field in the end") + { + REQUIRE_THROWS_AS( + getXDRFieldValidated(accountEntry, {"data", "account"}), + XDRQueryError); + } + } +} + +TEST_CASE("XDR matcher", "[xdrquery]") +{ + std::vector entries = { + makeAccountEntry(100), makeAccountEntry(200), makeOfferEntry("foo"), + makeOfferEntry("foobar")}; + entries[1].data.account().inflationDest.reset(); + + auto testMatches = [&](std::string const& query, + std::vector const& expectedMatches) { + XDRMatcher matcher(query); + for (int i = 0; i < expectedMatches.size(); ++i) + { + REQUIRE(matcher.matchXDR(entries[i]) == expectedMatches[i]); + } + }; + + SECTION("single comparison") + { + SECTION("ints") + { + testMatches("data.account.balance == 100", {true, false}); + testMatches("100 != data.account.balance", {false, true}); + testMatches("data.account.balance < 150", {true, false}); + testMatches("data.account.balance <= 100", {true, false}); + testMatches("data.account.balance > 150", {false, true}); + testMatches("200 >= data.account.balance", {true, true}); + } + + SECTION("strings") + { + testMatches("data.type == 'ACCOUNT'", {true, true, false, false}); + testMatches("data.type != 'ACCOUNT'", {false, false, true, true}); + testMatches("data.offer.selling.assetCode < 'foobar'", + {false, false, true, false}); + testMatches("data.offer.selling.assetCode <= 'foo'", + {false, false, true, false}); + testMatches("data.offer.selling.assetCode > 'foo'", + {false, false, false, true}); + testMatches("data.offer.selling.assetCode >= 'foo'", + {false, false, true, true}); + } + + SECTION("null") + { + testMatches("data.account.inflationDest == NULL", + {false, true, false, false}); + testMatches("NULL != data.account.inflationDest", + {true, false, false, false}); + } + } + + SECTION("queries with operators") + { + SECTION("or operator") + { + testMatches(R"( + data.account.accountID == "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG6ELY" + || data.offer.selling.issuer == "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG6ELY" + )", + {true, true, true, true}); + testMatches("data.account.balance > 150 || " + "data.offer.selling.assetCode == 'foo'", + {false, true, true, false}); + } + + SECTION("and operator") + { + testMatches(R"(data.account.balance > 150 + && '01000200' == data.account.thresholds)", + {false, true, false, false}); + testMatches("data.offer.selling.assetCode == 'foo' && data.type != " + "'TRUSTLINE'", + {false, false, true, false}); + } + + SECTION("mixed operators") + { + testMatches(R"(data.type != 'TRUSTLINE' && + ("01000200" == data.account.thresholds || + data.offer.selling.assetCode <= 'foo'))", + {true, true, true, false}); + testMatches(R"(("01000200" == data.account.thresholds || + data.offer.selling.assetCode <= 'foo') + && data.type != 'TRUSTLINE')", + {true, true, true, false}); + testMatches(R"("01000200" == data.account.thresholds || + data.type != 'TRUSTLINE' && + data.offer.selling.assetCode <= 'foo')", + {true, true, true, false}); + + testMatches(R"("01000200" == data.account.thresholds && + data.type != 'TRUSTLINE' && + data.offer.selling.assetCode <= 'foo')", + {false, false, false, false}); + testMatches(R"("01000200" == data.account.thresholds || + data.type != 'TRUSTLINE' || + data.offer.selling.assetCode <= 'foo')", + {true, true, true, true}); + } + } + + auto runQuery = [&](std::string const& query) { + XDRMatcher matcher(query); + matcher.matchXDR(entries[0]); + }; + SECTION("query errors") + { + SECTION("syntax error") + { + REQUIRE_THROWS_AS(runQuery("data.type == 'ACCOUNT"), XDRQueryError); + REQUIRE_THROWS_AS(runQuery("data.type = 'ACCOUNT'"), XDRQueryError); + REQUIRE_THROWS_AS(runQuery("$data.type == 'ACCOUNT'"), + XDRQueryError); + } + + SECTION("field error") + { + REQUIRE_THROWS_AS(runQuery("data.type.foo == 'ACCOUNT'"), + XDRQueryError); + REQUIRE_THROWS_AS(runQuery("data.account == 'ACCOUNT'"), + XDRQueryError); + REQUIRE_THROWS_AS(runQuery("data.account.accountID2 == 'ACCOUNT'"), + XDRQueryError); + REQUIRE_THROWS_AS(runQuery("data.account2.accountID == 'ACCOUNT'"), + XDRQueryError); + REQUIRE_THROWS_AS(runQuery("data2.account.accountID == 'ACCOUNT'"), + XDRQueryError); + REQUIRE_THROWS_AS(runQuery("account.accountID == 'ACCOUNT'"), + XDRQueryError); + } + + SECTION("type mismatch") + { + REQUIRE_THROWS_AS(runQuery("data.type == 123"), XDRQueryError); + REQUIRE_THROWS_AS(runQuery("data.account == 123"), XDRQueryError); + REQUIRE_THROWS_AS(runQuery("data.account.balance == '123'"), + XDRQueryError); + } + + SECTION("int out of range") + { + REQUIRE_THROWS_AS( + runQuery("data.account.balance <= 10000000000000000000"), + XDRQueryError); + REQUIRE_THROWS_AS( + runQuery("5000000000 > data.account.numSubEntries"), + XDRQueryError); + } + + SECTION("non-equality NULL comparison") + { + REQUIRE_THROWS_AS(runQuery("data.account.inflationDest <= NULL"), + XDRQueryError); + } + } +} + +} // namespace +} // namespace xdrquery