Runtime light: RPC support (#1030)

Add preliminary support for RPC into runtime-light.

Also, the PR fixes a bug in unsycnhronized_memory_resource.
It prevents defragmentation from merging two memory pieces from
distinct memory arenas, e.g. main memory resource and extra memory pool.

builtin-functions/kphp-light/functions.txt

@@ -76,6 +76,53 @@ function get_hash_of_class (object $klass) ::: int;
 
 function strlen ($str ::: string) ::: int;
 
+// === Rpc ========================================================================================
+
+/** @kphp-tl-class */
+interface RpcFunction {
+  public function getTLFunctionName() : string;
+}
+
+/** @kphp-tl-class */
+interface RpcFunctionReturnResult {}
+
+// type ReqResult <=> RpcResponse
+/** @kphp-tl-class */
+interface RpcResponse {
+  public function getResult() : @tl\RpcFunctionReturnResult;
+  public function getHeader() : @tl\_common\Types\rpcResponseHeader;
+  public function getError() : @tl\_common\Types\rpcResponseError;
+  public function isError() : bool;
+}
+
+/**
+ * 'KphpRpcRequestsExtraInfo' is a builtin KPHP class. It may accumulate extra information
+ * about RPC requests sent in both typed and untyped versions of rpc_tl_query builtins.
+ */
+final class KphpRpcRequestsExtraInfo {
+    /**
+    * 'get' returns an array of extra information (request size) about sent RPC requests.
+    *
+    * @return tuple(int)[]
+    */
+    public function get ();
+}
+
+/** @kphp-extern-func-info interruptible */
+function rpc_tl_query($actor ::: string, $arr ::: array, $timeout ::: float = -1.0, $ignore_answer ::: bool = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, $need_responses_extra_info ::: bool = false) ::: int[];
+
+/** @kphp-extern-func-info interruptible */
+function typed_rpc_tl_query($actor ::: string, @tl\RpcFunction[] $query_functions, $timeout ::: float = -1.0, $ignore_answer ::: bool = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, $need_responses_extra_info ::: bool = false) ::: int[];
+
+/** @kphp-extern-func-info interruptible */
+function rpc_tl_query_result($query_ids ::: array) ::: mixed[][];
+
+/** @kphp-extern-func-info interruptible */
+function typed_rpc_tl_query_result(int[] $query_ids) ::: @tl\RpcResponse[];
+
+
+// === Component ==================================================================================
+
 class ComponentQuery {
     private function __construct() ::: \ComponentQuery;
 }
@@ -118,6 +165,34 @@ function component_stream_read_exact($stream ::: ComponentStream, $len ::: int)
 function component_close_stream($stream ::: ComponentStream) ::: void;
 function component_finish_stream_processing($stream ::: ComponentStream) ::: void;
 
+// === Json =======================================================================================
+
+class JsonEncoder {
+  const rename_policy = 'none';
+  const visibility_policy = 'all';
+  const skip_if_default = false;
+  const float_precision = 0;
+
+  private function __construct();
+
+  public static function encode(object $instance, int $flags = 0, array $more = []) : string;
+  public static function decode(string $json, string $class_name) : instance<^2>;
+  public static function getLastError() : string;
+
+  // JsonEncoderOrChild::encode(...) is actually replaced by JsonEncoder::to_json_impl('JsonEncoderOrChild', ...)
+  static function to_json_impl(string $encoder_tag, object $instance, int $flags = 0, array $more = []) ::: string;
+
+  // JsonEncoderOrChild::decode(...) is actually replaced by JsonEncoder::from_json_impl('JsonEncoderOrChild', ...)
+  /** @kphp-extern-func-info cpp_template_call */
+  static function from_json_impl(string $encoder_tag, string $json, string $class_name) ::: instance<^3>;
+}
+
+function json_encode ($v ::: mixed, $options ::: int = 0) ::: string | false;
+
+function json_decode ($v ::: string, $assoc ::: bool = false) ::: mixed;
+
+// === Misc =======================================================================================
+
 /** @kphp-extern-func-info cpp_template_call */
 function instance_cast(object $instance, $to_type ::: string) ::: instance<^2>;
 
@@ -131,9 +206,6 @@ function warning($message ::: string) ::: void;
 /** @kphp-no-return */
 function critical_error($message ::: string) ::: void;
 
-function json_encode ($v ::: mixed, $options ::: int = 0) ::: string | false;
-function json_decode ($v ::: string, $assoc ::: bool = false) ::: mixed;
-
 function debug_print_string($str ::: string) ::: void;
 
 function byte_to_int($str ::: string) ::: ?int;

cmake/init-compilation-flags.cmake

@@ -105,6 +105,9 @@ endif()
 
 add_compile_options(-Werror -Wall -Wextra -Wunused-function -Wfloat-conversion -Wno-sign-compare
                     -Wuninitialized -Wno-redundant-move -Wno-missing-field-initializers)
+if(COMPILE_RUNTIME_LIGHT)
+    add_compile_options(-Wno-vla-cxx-extension)
+endif()
 
 if(NOT APPLE)
     check_cxx_compiler_flag(-gz=zlib DEBUG_COMPRESSION_IS_FOUND)

compiler/code-gen/files/init-scripts.cpp

@@ -238,7 +238,6 @@ void InitScriptsCpp::compile(CodeGenerator &W) const {
     }
   }
 
-
   W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL;
 
   if (G->is_output_mode_k2_component()) {

compiler/code-gen/files/tl2cpp/tl-combinator.cpp

@@ -66,7 +66,7 @@ void CombinatorStore::gen_before_args_processing(CodeGenerator &W) const {
   W << "(void)tl_object;" << NL;
   if (combinator->is_function()) {
     W << fmt_format("f$store_int({:#010x});", static_cast<unsigned int>(combinator->id)) << NL;
-    W << fmt_format("CurrentProcessingQuery::get().set_last_stored_tl_function_magic({:#010x});", static_cast<unsigned int>(combinator->id)) << NL;
+    W << fmt_format("CurrentTlQuery::get().set_last_stored_tl_function_magic({:#010x});", static_cast<unsigned int>(combinator->id)) << NL;
   }
 }
 
@@ -84,7 +84,7 @@ void CombinatorStore::gen_arg_processing(CodeGenerator &W, const std::unique_ptr
       if (!value_check.empty()) {
         W << "if (" << value_check << ") " << BEGIN;
         W
-          << fmt_format(R"(CurrentProcessingQuery::get().raise_storing_error("Optional field %s of %s is not set, but corresponding fields mask bit is set", "{}", "{}");)",
+          << fmt_format(R"(CurrentTlQuery::get().raise_storing_error("Optional field %s of %s is not set, but corresponding fields mask bit is set", "{}", "{}");)",
                         arg->name, combinator->name) << NL;
         W << "return" << (combinator->is_function() ? " {};" : ";") << NL;
         W << END << NL;
@@ -100,21 +100,22 @@ void CombinatorStore::gen_arg_processing(CodeGenerator &W, const std::unique_ptr
     auto *as_type_var = arg->type_expr->as<vk::tlo_parsing::type_var>();
     kphp_assert(as_type_var);
     if (!typed_mode) {
+      const auto *k2_tl_storers_prefix = G->is_output_mode_k2_component() ? "RpcImageState::get()." : "";
       W << "auto _cur_arg = "
         << fmt_format("tl_arr_get(tl_object, {}, {}, {}L)", tl2cpp::register_tl_const_str(arg->name), arg->idx, tl2cpp::hash_tl_const_str(arg->name))
         << ";" << NL;
       W << "string target_f_name = "
         << fmt_format("tl_arr_get(_cur_arg, {}, 0, {}L).as_string()", tl2cpp::register_tl_const_str("_"), tl2cpp::hash_tl_const_str("_"))
         << ";" << NL;
-      W << "if (!tl_storers_ht.has_key(target_f_name)) " << BEGIN
-        << "CurrentProcessingQuery::get().raise_storing_error(\"Function %s not found in tl-scheme\", target_f_name.c_str());" << NL
+      W << fmt_format("if (!{}tl_storers_ht.has_key(target_f_name)) ", k2_tl_storers_prefix) << BEGIN
+        << "CurrentTlQuery::get().raise_storing_error(\"Function %s not found in tl-scheme\", target_f_name.c_str());" << NL
         << "return {};" << NL
         << END << NL;
-      W << "const auto &storer_kv = tl_storers_ht.get_value(target_f_name);" << NL;
+      W << fmt_format("const auto &storer_kv = {}tl_storers_ht.get_value(target_f_name);", k2_tl_storers_prefix) << NL;
       W << "tl_func_state->" << combinator->get_var_num_arg(as_type_var->var_num)->name << ".fetcher = storer_kv(_cur_arg);" << NL;
     } else {
       W << "if (tl_object->$" << arg->name << ".is_null()) " << BEGIN
-        << R"(CurrentProcessingQuery::get().raise_storing_error("Field \")" << arg->name << R"(\" not found in tl object");)" << NL
+        << R"(CurrentTlQuery::get().raise_storing_error("Field \")" << arg->name << R"(\" not found in tl object");)" << NL
         << "return {};" << NL
         << END << NL;
       W << "tl_func_state->" << combinator->get_var_num_arg(as_type_var->var_num)->name << ".fetcher = "

compiler/code-gen/files/tl2cpp/tl-combinator.h

@@ -59,7 +59,7 @@ struct CombinatorGen {
       auto _cur_arg = tl_arr_get(tl_object, tl_str$query, 4, 1563700686);
       string target_f_name = tl_arr_get(_cur_arg, tl_str$_, 0, -2147483553).as_string();
       if (!tl_storers_ht.has_key(target_f_name)) {
-        CurrentProcessingQuery::get().raise_storing_error("Function %s not found in tl-scheme", target_f_name.c_str());
+        CurrentTlQuery::get().raise_storing_error("Function %s not found in tl-scheme", target_f_name.c_str());
         return {};
       }
       const auto &storer_kv = tl_storers_ht.get_value(target_f_name);

compiler/code-gen/files/tl2cpp/tl-function.cpp

@@ -69,18 +69,18 @@ void TlFunctionDef::compile(CodeGenerator &W) const {
   }
   if (f->is_kphp_rpc_server_function() && needs_typed_fetch_store) {
     FunctionSignatureGenerator(W) << "std::unique_ptr<tl_func_base> " << struct_name << "::rpc_server_typed_fetch(" << get_php_runtime_type(f) << " *tl_object) " << BEGIN;
-    W << "CurrentProcessingQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL;
+    W << "CurrentTlQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL;
     W << "auto tl_func_state = make_unique_on_script_memory<" << struct_name << ">();" << NL;
     W << CombinatorFetch(f, CombinatorPart::LEFT, true);
-    W << "CurrentProcessingQuery::get().reset();" << NL;
+    W << "CurrentTlQuery::get().reset();" << NL;
     W << "return std::move(tl_func_state);" << NL;
     W << END << NL << NL;
     FunctionSignatureGenerator(W) << "void " << struct_name << "::rpc_server_typed_store(const class_instance<" << G->settings().tl_classname_prefix.get()
                                   << "RpcFunctionReturnResult> &tl_object_) " << BEGIN;
-    W << "CurrentProcessingQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL;
+    W << "CurrentTlQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL;
     W << "auto tl_object = tl_object_.template cast_to<" << get_php_runtime_type(f, false) << "_result>().get();" << NL;
     W << CombinatorStore(f, CombinatorPart::RIGHT, true);
-    W << "CurrentProcessingQuery::get().reset();" << NL;
+    W << "CurrentTlQuery::get().reset();" << NL;
     W << END << NL << NL;
   }
 }

compiler/code-gen/files/tl2cpp/tl-type.cpp

@@ -20,7 +20,7 @@ void TypeStore::compile(CodeGenerator &W) const {
 
   if (typed_mode) {
     W << "if (tl_object.is_null()) " << BEGIN
-      << "CurrentProcessingQuery::get().raise_storing_error(\"Instance expected, but false given while storing tl type\");" << NL
+      << "CurrentTlQuery::get().raise_storing_error(\"Instance expected, but false given while storing tl type\");" << NL
       << "return;" << NL
       << END << NL;
   }
@@ -48,7 +48,7 @@ void TypeStore::compile(CodeGenerator &W) const {
       W << cpp_tl_struct_name("c_", c->name, template_str) << "::" << store_call << NL << END;
     }
     W << (first ? "" : " else ") << BEGIN
-      << "CurrentProcessingQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", "
+      << "CurrentTlQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", "
       << "tl_object.get_class(), \"" << type->name << "\");" << NL
       << END << NL;
 
@@ -68,7 +68,7 @@ void TypeStore::compile(CodeGenerator &W) const {
       W << cpp_tl_struct_name("c_", c->name, template_str) << "::" << store_call << NL << END;
     }
     W << (first ? "" : " else ") << BEGIN
-      << "CurrentProcessingQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", "
+      << "CurrentTlQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", "
       << "c_name.c_str(), \"" << type->name << "\");" << NL
       << END << NL;
   }
@@ -112,7 +112,7 @@ void TypeFetch::compile(CodeGenerator &W) const {
   if (default_constructor != nullptr) {
     W << "int pos = tl_parse_save_pos();" << NL;
   }
-  W << "auto magic = static_cast<unsigned int>(rpc_fetch_int());" << NL;
+  W << "auto magic = static_cast<unsigned int>(f$fetch_int());" << NL;
   W << "switch(magic) " << BEGIN;
   for (const auto &c : type->constructors) {
     if (c.get() == default_constructor) {
@@ -150,7 +150,7 @@ void TypeFetch::compile(CodeGenerator &W) const {
       W << "tl_object = result;" << NL;
     }
   } else {
-    W << "CurrentProcessingQuery::get().raise_fetching_error(\"Incorrect magic of type " << type->name << ": 0x%08x\", magic);" << NL;
+    W << "CurrentTlQuery::get().raise_fetching_error(\"Incorrect magic of type " << type->name << ": 0x%08x\", magic);" << NL;
   }
   W << END << NL;
   W << END << NL;

compiler/code-gen/files/tl2cpp/tl-type.h

@@ -22,7 +22,7 @@ void t_Either<T0, inner_magic0, T1, inner_magic1>::store(const mixed &tl_object)
     f$store_int(0xdf3ecb3b);
     c_right<T0, inner_magic0, T1, inner_magic1>::store(tl_object, std::move(X), std::move(Y));
   } else {
-    CurrentProcessingQuery::get().raise_storing_error("Invalid constructor %s of type %s", c_name.c_str(), "Either");
+    CurrentTlQuery::get().raise_storing_error("Invalid constructor %s of type %s", c_name.c_str(), "Either");
   }
 }
  * Typed TL:
@@ -37,7 +37,7 @@ void t_Either<T0, inner_magic0, T1, inner_magic1>::typed_store(const PhpType &tl
     const typename right__<typename T0::PhpType, typename T1::PhpType>::type *conv_obj = tl_object.template cast_to<typename right__<typename T0::PhpType, typename T1::PhpType>::type>().get();
     c_right<T0, inner_magic0, T1, inner_magic1>::typed_store(conv_obj, std::move(X), std::move(Y));
   } else {
-    CurrentProcessingQuery::get().raise_storing_error("Invalid constructor %s of type %s", tl_object.get_class(), "Either");
+    CurrentTlQuery::get().raise_storing_error("Invalid constructor %s of type %s", tl_object.get_class(), "Either");
   }
 }
 */
@@ -75,7 +75,7 @@ array<mixed> t_Either<T0, inner_magic0, T1, inner_magic1>::fetch() {
       break;
     }
     default: {
-      CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic);
+      CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic);
     }
   }
   return result;
@@ -101,7 +101,7 @@ void t_Either<T0, inner_magic0, T1, inner_magic1>::typed_fetch_to(PhpType &tl_ob
       break;
     }
     default: {
-      CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic);
+      CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic);
     }
   }
 }

compiler/code-gen/files/tl2cpp/tl2cpp.cpp

@@ -4,8 +4,9 @@
 
 #include "compiler/code-gen/files/tl2cpp/tl2cpp.h"
 
-#include "common/tlo-parsing/tlo-parsing.h"
+#include <sstream>
 
+#include "common/tlo-parsing/tlo-parsing.h"
 #include "compiler/code-gen/files/tl2cpp/tl-module.h"
 #include "compiler/code-gen/files/tl2cpp/tl2cpp-utils.h"
 #include "compiler/code-gen/naming.h"
@@ -89,7 +90,7 @@ void write_rpc_server_functions(CodeGenerator &W) {
   W << deps << NL;
   W << ExternInclude{G->settings().runtime_headers.get()} << NL;
   FunctionSignatureGenerator(W) << "class_instance<C$VK$TL$RpcFunction> f$rpc_server_fetch_request() " << BEGIN;
-  W << "auto function_magic = static_cast<unsigned int>(rpc_fetch_int());" << NL;
+  W << "auto function_magic = static_cast<unsigned int>(f$fetch_int());" << NL;
   W << "switch(function_magic) " << BEGIN;
   for (const auto &f : kphp_functions) {
     W << fmt_format("case {:#010x}: ", static_cast<unsigned int>(f->id)) << BEGIN;
@@ -147,13 +148,20 @@ void write_tl_query_handlers(CodeGenerator &W) {
     // a hash table that contains all TL functions;
     // it's passed to the runtime just like the fetch wrapper
     W << "array<tl_storer_ptr> gen$tl_storers_ht;" << NL;
+    // count the number of TL storers
+
     FunctionSignatureGenerator(W) << "void fill_tl_storers_ht() " << BEGIN;
+    std::stringstream ss{}; // TODO: use std::transform_reduce or something like that
+    int32_t tl_storers_nums = 0;
     for (const auto &module_name : modules_with_functions) {
+      tl_storers_nums += modules[module_name].target_functions.size();
       for (const auto &f : modules[module_name].target_functions) {
-        W << "gen$tl_storers_ht.set_value(" << register_tl_const_str(f->name) << ", " << "&" << cpp_tl_struct_name("f_", f->name) << "::store, "
-          << hash_tl_const_str(f->name) << "L);" << NL;
+        ss << "gen$tl_storers_ht.set_value(" << register_tl_const_str(f->name) << ", " << "&" << cpp_tl_struct_name("f_", f->name) << "::store, "
+           << hash_tl_const_str(f->name) << "L);\n";
       }
     }
+    W << fmt_format("gen$tl_storers_ht.reserve({}, false);", tl_storers_nums) << NL;
+    W << ss.str();
     W << END << NL;
     W << CloseFile();
   }

runtime-core/allocator/script-allocator-managed.h

@@ -1,3 +1,7 @@
+//  Compiler for PHP (aka KPHP)
+//  Copyright (c) 2024 LLC «V Kontakte»
+//  Distributed under the GPL v3 License, see LICENSE.notice.txt
+
 #pragma once
 
 #include <cstddef>

runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp

@@ -5,13 +5,15 @@
 #include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h"
 
 #include <algorithm>
+#include <cstdint>
 #include <functional>
 
 namespace memory_resource {
 namespace details {
 
-memory_ordered_chunk_list::memory_ordered_chunk_list(char *memory_resource_begin) noexcept:
-  memory_resource_begin_(memory_resource_begin) {
+memory_ordered_chunk_list::memory_ordered_chunk_list(char *memory_resource_begin, char *memory_resource_end) noexcept
+  : memory_resource_begin_(memory_resource_begin)
+  , memory_resource_end_(memory_resource_end) {
   static_assert(sizeof(list_node) == 8, "8 bytes expected");
 }
 
@@ -33,7 +35,12 @@ void memory_ordered_chunk_list::add_from_array(list_node **first, list_node **la
     return;
   }
 
+  last = std::partition(first, last, [this](const auto *mem) {
+    return reinterpret_cast<uintptr_t>(mem) >= reinterpret_cast<uintptr_t>(this->memory_resource_begin_)
+           && reinterpret_cast<uintptr_t>(mem) < reinterpret_cast<uintptr_t>(this->memory_resource_end_);
+  });
   std::sort(first, last, std::greater<>{});
+
   if (!head_) {
     head_ = *first++;
   } else if (reinterpret_cast<char *>(head_) < reinterpret_cast<char *>(*first)) {

runtime-core/memory-resource/details/memory_ordered_chunk_list.h

@@ -34,7 +34,7 @@ class memory_ordered_chunk_list : vk::not_copyable {
     uint32_t chunk_size_{0};
   };
 
-  explicit memory_ordered_chunk_list(char *memory_resource_begin) noexcept;
+  explicit memory_ordered_chunk_list(char *memory_resource_begin, char *memory_resource_end) noexcept;
 
   list_node *get_next(const list_node *node) const noexcept {
     return node->has_next() ? reinterpret_cast<list_node *>(memory_resource_begin_ + node->next_chunk_offset_) : nullptr;
@@ -56,6 +56,7 @@ class memory_ordered_chunk_list : vk::not_copyable {
   void add_from_array(list_node **first, list_node **last) noexcept;
 
   char *memory_resource_begin_{nullptr};
+  char *memory_resource_end_{nullptr};
   list_node *head_{nullptr};
   size_t tmp_buffer_size_{0};
   std::array<list_node *, 4096> tmp_buffer_;

runtime-core/memory-resource/extra-memory-pool.h

@@ -6,8 +6,8 @@
 
 #include <array>
 #include <cassert>
-#include <cinttypes>
 #include <cstddef>
+#include <cstdint>
 
 #include "common/mixin/not_copyable.h"
 
@@ -24,8 +24,8 @@ class alignas(8) extra_memory_pool : vk::not_copyable {
   }
 
   bool is_memory_from_this_pool(const void *mem, size_t mem_size) noexcept {
-    return memory_begin() <= static_cast<const uint8_t *>(mem) &&
-           static_cast<const uint8_t *>(mem) + mem_size <= memory_begin() + get_pool_payload_size();
+    return reinterpret_cast<uintptr_t>(memory_begin()) <= reinterpret_cast<uintptr_t>(mem)
+           && reinterpret_cast<uintptr_t>(mem) + mem_size <= reinterpret_cast<uintptr_t>(memory_begin()) + get_pool_payload_size();
   }
 
   static size_t get_pool_payload_size(size_t buffer_size) noexcept {