Merge pull request stellar#3447 from dmkozh/ledger_dump

Added utility to dump ledger state from buckets to JSON

Reviewed-by: MonsieurNicolas

.gitignore

@@ -104,3 +104,6 @@ min-testcases/
 *.vcxproj.user
 **/.vs/*
 
+/src/util/xdrquery/XDRQueryScanner.cpp
+/src/util/xdrquery/XDRQueryParser.h
+/src/util/xdrquery/XDRQueryParser.cpp

Builds/VisualStudio/stellar-core.vcxproj

@@ -130,7 +130,7 @@
       </ControlFlowGuard>
       <SmallerTypeCheck>false</SmallerTypeCheck>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/bigobj /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
@@ -188,7 +188,7 @@ exit /b 0
       </ControlFlowGuard>
       <SmallerTypeCheck>false</SmallerTypeCheck>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/bigobj /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
@@ -247,7 +247,7 @@ exit /b 0
       </ControlFlowGuard>
       <SmallerTypeCheck>false</SmallerTypeCheck>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/bigobj /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
@@ -305,7 +305,7 @@ exit /b 0
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <DisableSpecificWarnings>4060;4100;4127;4324;4408;4510;4512;4582;4583;4592</DisableSpecificWarnings>
       <ControlFlowGuard>false</ControlFlowGuard>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/bigobj /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
       <OmitFramePointers>false</OmitFramePointers>
     </ClCompile>
@@ -363,7 +363,7 @@ exit /b 0
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <DisableSpecificWarnings>4060;4100;4127;4324;4408;4510;4512;4582;4583;4592</DisableSpecificWarnings>
       <ControlFlowGuard>false</ControlFlowGuard>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/bigobj /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
       <OmitFramePointers>false</OmitFramePointers>
     </ClCompile>
@@ -543,6 +543,11 @@ exit /b 0
     <ClCompile Include="..\..\src\util\test\MetricTests.cpp" />
     <ClCompile Include="..\..\src\util\test\SchedulerTests.cpp" />
     <ClCompile Include="..\..\src\util\XDRCereal.cpp" />
+    <ClCompile Include="..\..\src\util\xdrquery\tests\XDRQueryTests.cpp" />
+    <ClCompile Include="..\..\src\util\xdrquery\XDRMatcher.cpp" />
+    <ClCompile Include="..\..\src\util\xdrquery\XDRQueryEval.cpp" />
+    <ClCompile Include="..\..\src\util\xdrquery\XDRQueryParser.cpp" />
+    <ClCompile Include="..\..\src\util\xdrquery\XDRQueryScanner.cpp" />
     <ClCompile Include="..\..\src\work\BatchWork.cpp" />
     <ClCompile Include="..\..\src\historywork\CheckSingleLedgerHeaderWork.cpp" />
     <ClCompile Include="..\..\src\historywork\DownloadBucketsWork.cpp" />
@@ -939,6 +944,11 @@ exit /b 0
     <ClInclude Include="..\..\src\util\UnorderedSet.h" />
     <ClInclude Include="..\..\src\util\XDRCereal.h" />
     <ClInclude Include="..\..\src\util\XDROperators.h" />
+    <ClInclude Include="..\..\src\util\xdrquery\XDRMatcher.h" />
+    <ClInclude Include="..\..\src\util\xdrquery\XDRFieldResolver.h" />
+    <ClInclude Include="..\..\src\util\xdrquery\XDRQueryEval.h" />
+    <ClInclude Include="..\..\src\util\xdrquery\XDRQueryError.h" />
+    <ClInclude Include="..\..\src\util\xdrquery\XDRQueryParser.h" />
     <ClInclude Include="..\..\src\work\BatchWork.h" />
     <ClInclude Include="..\..\src\historywork\CheckSingleLedgerHeaderWork.h" />
     <ClInclude Include="..\..\src\historywork\DownloadBucketsWork.h" />
@@ -1444,6 +1454,20 @@ $(OutDir)\bin\cxxbridge.exe ..\..\src\rust\src\lib.rs --output src\$(Configurati
     <None Include="..\..\src\transactions\readme.md" />
     <None Include="..\..\ci-build.sh" />
     <None Include="..\..\src\util\LogPartitions.def" />
+    <CustomBuild Include="..\..\src\util\xdrquery\XDRQueryParser.yy">
+      <FileType>Document</FileType>
+      <Command>bison --defines=../../src/util/xdrquery/XDRQueryParser.h  --output=../../src/util/xdrquery/XDRQueryParser.cpp ../../src/util/xdrquery/XDRQueryParser.yy</Command>
+      <Message>running bison for XDRQueryParser.yy</Message>
+      <Outputs>../../src/util/xdrquery/XDRQueryParser.h</Outputs>
+      <BuildInParallel>true</BuildInParallel>
+    </CustomBuild>
+    <CustomBuild Include="..\..\src\util\xdrquery\XDRQueryScanner.ll">
+      <FileType>Document</FileType>
+      <Command>flex --outfile=../../src/util/xdrquery/XDRQueryScanner.cpp ../../src/util/xdrquery/XDRQueryScanner.ll</Command>
+      <Message>running flex for XDRQueryScanner.ll</Message>
+      <Outputs>../../src/util/xdrquery/XDRQueryScanner.cpp</Outputs>
+      <BuildInParallel>true</BuildInParallel>
+    </CustomBuild>
     <None Include="libmedida\libmedida.vcxproj" />
   </ItemGroup>
   <ItemGroup>

Builds/VisualStudio/stellar-core.vcxproj.filters

@@ -172,6 +172,12 @@
     <Filter Include="rust\generated">
       <UniqueIdentifier>{57d45a48-6030-4dff-96ce-fd587f319529}</UniqueIdentifier>
     </Filter>
+    <Filter Include="util\xdrquery">
+      <UniqueIdentifier>{166a403b-4f13-4505-9953-97ea92e5d855}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="util\xdrquery\tests">
+      <UniqueIdentifier>{60ef25c4-d908-4795-83cf-a348673fe4c4}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\lib\util\getopt_long.c">
@@ -1254,6 +1260,21 @@
     <ClCompile Include="..\..\src\herder\TxSetUtils.cpp">
       <Filter>herder</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\util\xdrquery\XDRMatcher.cpp">
+      <Filter>util\xdrquery</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\util\xdrquery\XDRQueryEval.cpp">
+      <Filter>util\xdrquery</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\util\xdrquery\XDRQueryParser.cpp">
+      <Filter>util\xdrquery</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\util\xdrquery\XDRQueryScanner.cpp">
+      <Filter>util\xdrquery</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\util\xdrquery\tests\XDRQueryTests.cpp">
+      <Filter>util\xdrquery\tests</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\lib\util\cpptoml.h">
@@ -2183,6 +2204,21 @@
     <ClInclude Include="src\$(Configuration)\generated\xdr\Stellar-types.h">
       <Filter>xdr\generated</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\util\xdrquery\XDRMatcher.h">
+      <Filter>util\xdrquery</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\util\xdrquery\XDRFieldResolver.h">
+      <Filter>util\xdrquery</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\util\xdrquery\XDRQueryEval.h">
+      <Filter>util\xdrquery</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\util\xdrquery\XDRQueryError.h">
+      <Filter>util\xdrquery</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\util\xdrquery\XDRQueryParser.h">
+      <Filter>util\xdrquery</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\AUTHORS" />
@@ -2374,6 +2410,12 @@
     <CustomBuild Include="..\..\src\protocol-curr\xdr\Stellar-SCP.x">
       <Filter>xdr\curr</Filter>
     </CustomBuild>
+    <CustomBuild Include="..\..\src\util\xdrquery\XDRQueryParser.yy">
+      <Filter>util\xdrquery</Filter>
+    </CustomBuild>
+    <CustomBuild Include="..\..\src\util\xdrquery\XDRQueryScanner.ll">
+      <Filter>util\xdrquery</Filter>
+    </CustomBuild>
   </ItemGroup>
   <ItemGroup>
     <Text Include="..\..\INSTALL-Windows.md" />

docs/software/commands.md

@@ -34,7 +34,33 @@ Command options can only by placed after command.
   private key. For example:
 
 `$ stellar-core convert-id SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2`
-
+* **dump-ledger**: Dumps the current ledger state from bucket files into
+    JSON **--output-file** with optional filtering. **--last-ledgers** option
+    allows to only dump the ledger entries that were last modified within that
+    many ledgers. **--limit** option limits the output to that many arbitrary
+    records. **--filter-query** allows to specify a filtering expression over
+    `LedgerEntry` XDR. Expression should evaluate to boolean and consist of
+    field paths, comparisons, literals, boolean operators (`&&`, ` ||`) and
+    parentheses. The field values are consistent with `print-xdr` JSON
+    representation: enums are represented as their name strings, account ids as
+    encoded strings, hashes as hex strings etc. Filtering is useful to minimize
+    the output JSON size and then optionally process it further with tools like
+    `jq`. Query examples:
+
+    * `data.type == 'OFFER'` - dump only offers
+    * `data.account.accountID == 'GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' || 
+       data.trustLine.accountID == "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"`
+       - dump only account and trustline entries for the specified account.
+    * `data.account.inflationDest != NULL` - dump accounts that have an optional
+      `inflationDest` field set.
+    * `data.offer.selling.assetCode == 'FOOBAR' &&
+       data.offer.selling.issuer == 'GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'` -
+       dump offers that are selling the specified asset.
+    * `data.trustLine.ext.v1.liabilities.buying < data.trustLine.ext.v1.liabilities.selling` -
+      dump trustlines that have buying liabilites less than selling liabilites
+    * `(data.account.balance < 100000000 || data.account.balance >= 2000000000) 
+       && data.account.numSubEntries > 2` - dump accounts with certain balance 
+       and sub entries count, demonstrates more complex expression
 * **dump-xdr <FILE-NAME>**:  Dumps the given XDR file and then exits.
 * **encode-asset --code <CODE> --issuer <ISSUER>**: Prints a base-64 encoded asset.
   Prints the native asset if neither `code` nor `issuer` is given.

src/Makefile.am

@@ -32,6 +32,21 @@ SUFFIXES = .x .h .rs
 .x.h:
 	$(XDRC) -hh -pedantic -o $@ $<
 
+BISON=bison
+FLEX=flex
+
+$(srcdir)/util/xdrquery/XDRQueryScanner.cpp: $(srcdir)/util/xdrquery/XDRQueryScanner.ll
+	$(FLEX) --outfile=$@ $<
+
+$(srcdir)/util/xdrquery/XDRQueryParser.cpp: $(srcdir)/util/xdrquery/XDRQueryParser.yy
+	$(BISON) --defines=$(srcdir)/util/xdrquery/XDRQueryParser.h  --output=$@ $<
+
+$(srcdir)/util/xdrquery/XDRQueryParser.h: $(srcdir)/util/xdrquery/XDRQueryParser.cpp
+	touch $@
+
+BUILT_SOURCES += $(srcdir)/util/xdrquery/XDRQueryScanner.cpp $(srcdir)/util/xdrquery/XDRQueryParser.h $(srcdir)/util/xdrquery/XDRQueryParser.cpp
+stellar_core_SOURCES += $(srcdir)/util/xdrquery/XDRQueryScanner.cpp $(srcdir)/util/xdrquery/XDRQueryParser.h $(srcdir)/util/xdrquery/XDRQueryParser.cpp
+
 # Old automakes have buggy dependency tracking for conditional generated
 # sources. We work around this here by making rust/RustBridge.{cpp,h} generated
 # in all cases, and just empty in the non-rust case. Also because of the way old

src/bucket/BucketManager.h

@@ -10,6 +10,7 @@
 #include <future>
 #include <map>
 #include <memory>
+#include <optional>
 #include <set>
 
 #include "medida/timer_context.h"
@@ -230,6 +231,28 @@ class BucketManager : NonMovableOrCopyable
     virtual std::shared_ptr<Bucket>
     mergeBuckets(HistoryArchiveState const& has) = 0;
 
+    // Visits all the active ledger entries or subset thereof.
+    //
+    // The order in which the entries are visited is not defined, but roughly
+    // goes from more fresh entries to the older ones.
+    //
+    // This accepts two visitors. `filterEntry` has to return `true`
+    // if the ledger entry can *potentially* be accepted. The passed entry isn't
+    // necessarily fresh or even alive. `acceptEntry` will only get the fresh
+    // alive entries that have passed the filter. If it returns `false` the
+    // iteration will immediately finish.
+    //
+    // When `minLedger` is specified, only entries that have been modified at
+    // `minLedger` or later are visited.
+    //
+    // When `filterEntry` and `acceptEntry` always return `true`, this is
+    // equivalent to iterating over `loadCompleteLedgerState`, so the same
+    // memory/runtime implications apply.
+    virtual void visitLedgerEntries(
+        HistoryArchiveState const& has, std::optional<int64_t> minLedger,
+        std::function<bool(LedgerEntry const&)> const& filterEntry,
+        std::function<bool(LedgerEntry const&)> const& acceptEntry) = 0;
+
     // Schedule a Work class that verifies the hashes of all referenced buckets
     // on background threads.
     virtual std::shared_ptr<BasicWork>

src/bucket/BucketManagerImpl.cpp

@@ -966,6 +966,116 @@ BucketManagerImpl::mergeBuckets(HistoryArchiveState const& has)
     return out.getBucket(*this);
 }
 
+static bool
+visitEntriesInBucket(std::shared_ptr<Bucket const> b, std::string const& name,
+                     std::optional<int64_t> minLedger,
+                     std::function<bool(LedgerEntry const&)> const& filterEntry,
+                     std::function<bool(LedgerEntry const&)> const& acceptEntry,
+                     UnorderedSet<LedgerKey>& processedEntries)
+{
+    using namespace std::chrono;
+    medida::Timer timer;
+
+    UnorderedMap<LedgerKey, LedgerEntry> bucketEntries;
+    bool stopIteration = false;
+    timer.Time([&]() {
+        for (BucketInputIterator in(b); in; ++in)
+        {
+            BucketEntry const& e = *in;
+            if (e.type() == LIVEENTRY || e.type() == INITENTRY)
+            {
+                if (minLedger &&
+                    e.liveEntry().lastModifiedLedgerSeq < *minLedger)
+                {
+                    stopIteration = true;
+                    continue;
+                }
+                if (!filterEntry(e.liveEntry()))
+                {
+                    continue;
+                }
+                auto key = LedgerEntryKey(e.liveEntry());
+                if (processedEntries.find(key) != processedEntries.end())
+                {
+                    continue;
+                }
+                bucketEntries[key] = e.liveEntry();
+            }
+            else
+            {
+                if (e.type() != DEADENTRY)
+                {
+                    std::string err = "Malformed bucket: unexpected "
+                                      "non-INIT/LIVE/DEAD entry.";
+                    CLOG_ERROR(Bucket, "{}", err);
+                    throw std::runtime_error(err);
+                }
+                bucketEntries.erase(e.deadEntry());
+                processedEntries.insert(e.deadEntry());
+            }
+        }
+        for (auto const& [key, entry] : bucketEntries)
+        {
+            processedEntries.insert(key);
+            if (!acceptEntry(entry))
+            {
+                stopIteration = true;
+                break;
+            }
+        }
+    });
+    nanoseconds ns =
+        timer.duration_unit() * static_cast<nanoseconds::rep>(timer.max());
+    milliseconds ms = duration_cast<milliseconds>(ns);
+    size_t bytesPerSec = (b->getSize() * 1000 / (1 + ms.count()));
+    CLOG_INFO(Bucket, "Processed {}-byte bucket file '{}' in {} ({}/s)",
+              b->getSize(), name, ms, formatSize(bytesPerSec));
+    return !stopIteration;
+}
+
+void
+BucketManagerImpl::visitLedgerEntries(
+    HistoryArchiveState const& has, std::optional<int64_t> minLedger,
+    std::function<bool(LedgerEntry const&)> const& filterEntry,
+    std::function<bool(LedgerEntry const&)> const& acceptEntry)
+{
+    UnorderedSet<LedgerKey> deletedEntries;
+    std::vector<std::pair<Hash, std::string>> hashes;
+    for (uint32_t i = 0; i < BucketList::kNumLevels; ++i)
+    {
+        HistoryStateBucket const& hsb = has.currentBuckets.at(i);
+        hashes.emplace_back(hexToBin256(hsb.curr),
+                            fmt::format(FMT_STRING("curr {:d}"), i));
+        hashes.emplace_back(hexToBin256(hsb.snap),
+                            fmt::format(FMT_STRING("snap {:d}"), i));
+    }
+    medida::Timer timer;
+    timer.Time([&]() {
+        for (auto const& pair : hashes)
+        {
+            if (isZero(pair.first))
+            {
+                continue;
+            }
+            auto b = getBucketByHash(pair.first);
+            if (!b)
+            {
+                throw std::runtime_error(std::string("missing bucket: ") +
+                                         binToHex(pair.first));
+            }
+            if (!visitEntriesInBucket(b, pair.second, minLedger, filterEntry,
+                                      acceptEntry, deletedEntries))
+            {
+                break;
+            }
+        }
+    });
+    auto ns = timer.duration_unit() *
+              static_cast<std::chrono::nanoseconds::rep>(timer.max());
+    CLOG_INFO(Bucket, "Total ledger processing time: {}",
+              std::chrono::duration_cast<std::chrono::milliseconds>(ns));
+}
+
 std::shared_ptr<BasicWork>
 BucketManagerImpl::scheduleVerifyReferencedBucketsWork()
 {

src/bucket/BucketManagerImpl.h

@@ -143,6 +143,11 @@ class BucketManagerImpl : public BucketManager
     std::shared_ptr<Bucket>
     mergeBuckets(HistoryArchiveState const& has) override;
 
+    void visitLedgerEntries(
+        HistoryArchiveState const& has, std::optional<int64_t> minLedger,
+        std::function<bool(LedgerEntry const&)> const& filterEntry,
+        std::function<bool(LedgerEntry const&)> const& acceptEntry) override;
+
     std::shared_ptr<BasicWork> scheduleVerifyReferencedBucketsWork() override;
 };