Merge pull request #1537 from vogel/cleanup-stale-buckets

Cleanup stale buckets

Reviewed-by: MonsieurNicolas

src/bucket/BucketList.cpp

@@ -313,6 +313,12 @@ BucketList::keepDeadEntries(uint32_t level)
     return level < BucketList::kNumLevels - 1;
 }
 
+BucketLevel const&
+BucketList::getLevel(uint32_t i) const
+{
+    return mLevels.at(i);
+}
+
 BucketLevel&
 BucketList::getLevel(uint32_t i)
 {

src/bucket/BucketList.h

@@ -328,6 +328,9 @@ class BucketList
     // an empty bucket in `curr` and `snap`.
     BucketList();
 
+    // Return level `i` of the BucketList.
+    BucketLevel const& getLevel(uint32_t i) const;
+
     // Return level `i` of the BucketList.
     BucketLevel& getLevel(uint32_t i);
 

src/bucket/BucketManagerImpl.cpp

@@ -16,6 +16,7 @@
 #include "util/types.h"
 #include <fstream>
 #include <map>
+#include <regex>
 #include <set>
 
 #include "medida/counter.h"
@@ -70,12 +71,28 @@ BucketManagerImpl::BucketManagerImpl(Application& app)
 
 const std::string BucketManagerImpl::kLockFilename = "stellar-core.lock";
 
-static std::string
+namespace
+{
+std::string
 bucketBasename(std::string const& bucketHexHash)
 {
     return "bucket-" + bucketHexHash + ".xdr";
 }
 
+bool
+isBucketFile(std::string const& name)
+{
+    static std::regex re("^bucket-[a-z0-9]{64}\\.xdr(\\.gz)?$");
+    return std::regex_match(name, re);
+};
+
+uint256
+extractFromFilename(std::string const& name)
+{
+    return hexToBin256(name.substr(7, 64));
+};
+}
+
 std::string
 BucketManagerImpl::bucketFilename(std::string const& bucketHexHash)
 {
@@ -211,11 +228,10 @@ BucketManagerImpl::getBucketByHash(uint256 const& hash)
     return std::shared_ptr<Bucket>();
 }
 
-void
-BucketManagerImpl::forgetUnreferencedBuckets()
+std::set<Hash>
+BucketManagerImpl::getReferencedBuckets() const
 {
-    std::lock_guard<std::recursive_mutex> lock(mBucketMutex);
-    std::set<Hash> referenced;
+    auto referenced = std::set<Hash>{};
     for (uint32_t i = 0; i < BucketList::kNumLevels; ++i)
     {
         auto const& level = mBucketList.getLevel(i);
@@ -240,6 +256,40 @@ BucketManagerImpl::forgetUnreferencedBuckets()
         }
     }
 
+    return referenced;
+}
+
+void
+BucketManagerImpl::cleanupStaleFiles()
+{
+    std::lock_guard<std::recursive_mutex> lock(mBucketMutex);
+    auto referenced = getReferencedBuckets();
+    std::transform(std::begin(mSharedBuckets), std::end(mSharedBuckets),
+                   std::inserter(referenced, std::end(referenced)),
+                   [](std::pair<Hash, std::shared_ptr<Bucket>> const& p) {
+                       return p.first;
+                   });
+
+    for (auto f : fs::findfiles(getBucketDir(), isBucketFile))
+    {
+        auto hash = extractFromFilename(f);
+        if (referenced.find(hash) == std::end(referenced))
+        {
+            // we don't care about failure here
+            // if removing file failed one time, it may not fail when this is
+            // called again
+            auto fullName = getBucketDir() + "/" + f;
+            std::remove(fullName.c_str());
+        }
+    }
+}
+
+void
+BucketManagerImpl::forgetUnreferencedBuckets()
+{
+    std::lock_guard<std::recursive_mutex> lock(mBucketMutex);
+    auto referenced = getReferencedBuckets();
+
     for (auto i = mSharedBuckets.begin(); i != mSharedBuckets.end();)
     {
         // Standard says map iterators other than the one you're erasing remain
@@ -268,6 +318,8 @@ BucketManagerImpl::forgetUnreferencedBuckets()
             {
                 CLOG(TRACE, "Bucket") << "removing bucket file: " << filename;
                 std::remove(filename.c_str());
+                auto gzfilename = filename + ".gz";
+                std::remove(gzfilename.c_str());
             }
             mSharedBuckets.erase(j);
         }
@@ -351,6 +403,7 @@ BucketManagerImpl::assumeState(HistoryArchiveState const& has)
     }
 
     mBucketList.restartMerges(mApp);
+    cleanupStaleFiles();
 }
 
 void

src/bucket/BucketManagerImpl.h

@@ -7,6 +7,7 @@
 #include <map>
 #include <memory>
 #include <mutex>
+#include <set>
 #include <string>
 
 // Copyright 2015 Stellar Development Foundation and contributors. Licensed
@@ -45,6 +46,9 @@ class BucketManagerImpl : public BucketManager
     medida::Timer& mBucketSnapMerge;
     medida::Counter& mSharedBucketsSize;
 
+    std::set<Hash> getReferencedBuckets() const;
+    void cleanupStaleFiles();
+
   protected:
     void calculateSkipValues(LedgerHeader& currentHeader);
     std::string bucketFilename(std::string const& bucketHexHash);

src/util/Fs.cpp

@@ -12,7 +12,9 @@
 
 #ifdef _WIN32
 #include <direct.h>
+#include <filesystem>
 #else
+#include <dirent.h>
 #include <sys/stat.h>
 #endif
 
@@ -118,6 +120,28 @@ deltree(std::string const& d)
     }
 }
 
+std::vector<std::string>
+findfiles(std::string const& p,
+          std::function<bool(std::string const& name)> predicate)
+{
+    using namespace std;
+    namespace fs = std::experimental::filesystem;
+
+    std::vector<std::string> res;
+    for (auto& entry : fs::directory_iterator(fs::path(p)))
+    {
+        if (fs::is_regular_file(entry.status()))
+        {
+            auto n = entry.path().filename().string();
+            if (predicate(n))
+            {
+                res.emplace_back(n);
+            }
+        }
+    }
+    return res;
+}
+
 long
 getCurrentPid()
 {
@@ -270,6 +294,38 @@ deltree(std::string const& d)
     }
 }
 
+std::vector<std::string>
+findfiles(std::string const& path,
+          std::function<bool(std::string const& name)> predicate)
+{
+    auto dir = opendir(path.c_str());
+    auto result = std::vector<std::string>{};
+    if (!dir)
+    {
+        return result;
+    }
+
+    try
+    {
+        while (auto entry = readdir(dir))
+        {
+            auto name = std::string{entry->d_name};
+            if (predicate(name))
+            {
+                result.push_back(name);
+            }
+        }
+
+        return result;
+    }
+    catch (...)
+    {
+        // small RAII class could do here
+        closedir(dir);
+        throw;
+    }
+}
+
 long
 getCurrentPid()
 {

src/util/Fs.h

@@ -4,7 +4,9 @@
 // under the Apache License, Version 2.0. See the COPYING file at the root
 // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
 
+#include <functional>
 #include <string>
+#include <vector>
 
 namespace stellar
 {
@@ -40,6 +42,12 @@ bool mkdir(std::string const& path);
 // Make a dir path like mkdir -p, i.e. recursive, uses '/' as dir separator
 bool mkpath(std::string const& path);
 
+// Get list of all files with names matching predicate
+// Returned names are relative to path
+std::vector<std::string>
+findfiles(std::string const& path,
+          std::function<bool(std::string const& name)> predicate);
+
 class PathSplitter
 {
   public: