Make master outlive confdata out of shared memory errors (#946)

- events throttling approach with soft/hard OOM mode
- external allocation heuristic checks to prevent OOM
- timeout for reading binlog on updates to prevent master freezing 
- new emergency option for blacklisting confdata events
- extend confdata monitoring in StatsHouse and Graphana 

Co-authored-by: Aleksandr Kirsanov <[email protected]>

runtime/array_decl.inl

@@ -252,6 +252,10 @@ public:
   // we can create true vector
   bool has_no_string_keys() const noexcept;
 
+  static size_t estimate_size(int64_t n, bool is_vector) noexcept {
+    return array_inner::estimate_size(n, is_vector);
+  }
+
   T &operator[](int64_t int_key);
   T &operator[](int32_t key) { return (*this)[int64_t{key}]; }
   T &operator[](const string &s);

runtime/confdata-global-manager.cpp

@@ -95,11 +95,12 @@ ConfdataGlobalManager &ConfdataGlobalManager::get() noexcept {
 
 void ConfdataGlobalManager::init(size_t confdata_memory_limit,
                                  std::unordered_set<vk::string_view> &&predefined_wilrdcards,
-                                 std::unique_ptr<re2::RE2> &&blacklist_pattern) noexcept {
+                                 std::unique_ptr<re2::RE2> &&blacklist_pattern,
+                                 std::forward_list<vk::string_view> &&force_ignore_prefixes) noexcept {
   resource_.init(mmap_shared(confdata_memory_limit), confdata_memory_limit);
   confdata_samples_.init(resource_);
   predefined_wildcards_.set_wildcards(std::move(predefined_wilrdcards));
-  key_blacklist_.set_blacklist(std::move(blacklist_pattern));
+  key_blacklist_.set_blacklist(std::move(blacklist_pattern), std::move(force_ignore_prefixes));
 }
 
 ConfdataGlobalManager::~ConfdataGlobalManager() noexcept {

runtime/confdata-global-manager.h

@@ -51,8 +51,9 @@ class ConfdataGlobalManager : vk::not_copyable {
   static ConfdataGlobalManager &get() noexcept;
 
   void init(size_t confdata_memory_limit,
-    std::unordered_set<vk::string_view> &&predefined_wilrdcards,
-            std::unique_ptr<re2::RE2> &&blacklist_pattern) noexcept;
+            std::unordered_set<vk::string_view> &&predefined_wilrdcards,
+            std::unique_ptr<re2::RE2> &&blacklist_pattern,
+            std::forward_list<vk::string_view> &&force_ignore_prefixes) noexcept;
 
   void force_release_all_resources_acquired_by_this_proc_if_init() noexcept {
     if (is_initialized()) {

runtime/confdata-keys.cpp

@@ -131,3 +131,12 @@ void ConfdataKeyMaker::forcibly_change_first_key_wildcard_dots_from_two_to_one()
   second_key_ = string::make_const_string_on_memory(first_key_end, second_key_len, second_key_buffer_.data(), second_key_buffer_.size());
   first_key_type_ = ConfdataFirstKeyType::one_dot_wildcard;
 }
+
+bool ConfdataKeyBlacklist::is_key_forcibly_ignored_by_prefix(vk::string_view key) const noexcept {
+  for (const vk::string_view &prefix : force_ignore_prefixes_) {
+    if (key.starts_with(prefix)) {
+      return true;
+    }
+  }
+  return false;
+}

runtime/confdata-keys.h

@@ -5,6 +5,7 @@
 #pragma once
 #include <array>
 #include <re2/re2.h>
+#include <forward_list>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -112,18 +113,31 @@ class ConfdataKeyMaker : vk::not_copyable {
 };
 
 class ConfdataKeyBlacklist : vk::not_copyable {
+  bool is_key_forcibly_ignored_by_prefix(vk::string_view key) const noexcept;
+
 public:
-  void set_blacklist(std::unique_ptr<re2::RE2> &&blacklist_pattern) noexcept {
+  void set_blacklist(std::unique_ptr<re2::RE2> &&blacklist_pattern, std::forward_list<vk::string_view> &&force_ignore_prefixes) noexcept {
     blacklist_pattern_ = std::move(blacklist_pattern);
+    force_ignore_prefixes_ = std::move(force_ignore_prefixes);
   }
 
   bool is_blacklisted(vk::string_view key) const noexcept {
-    return blacklist_pattern_ &&
+    // from PHP class KphpConfiguration, e.g. for langs
+    if (blacklist_pattern_ &&
            re2::RE2::FullMatch(
              re2::StringPiece(key.data(), static_cast<uint32_t>(key.size())),
-             *blacklist_pattern_);
+             *blacklist_pattern_)) {
+      return true;
+    }
+    // emergency startup option to disable keys by prefix, e.g. 'highload.vid'
+    if (unlikely(!force_ignore_prefixes_.empty()) &&
+        is_key_forcibly_ignored_by_prefix(key)) {
+      return true;
+    }
+    return false;
   }
 
 private:
   std::unique_ptr<re2::RE2> blacklist_pattern_;
+  std::forward_list<vk::string_view> force_ignore_prefixes_;
 };

server/confdata-binlog-events.h

@@ -90,6 +90,10 @@ struct lev_confdata_store_wrapper : impl_::confdata_event<BASE, impl_::select_st
     return size <= 0 ? mixed{string{}} : mc_get_value(mem, size, get_flags());
   }
 
+  size_t get_data_size() const noexcept {
+    return BASE::data_len;
+  }
+
   vk::string_view get_value_as_string() const noexcept {
     const char *mem = BASE::data + BASE::key_len;
     const int size = BASE::data_len;