From 38b4837706afd4628b5b8c9598920287d1089281 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Thu, 26 Sep 2024 14:09:50 +0300 Subject: [PATCH] Enhance K2 TL lib (#1109) * enhance TL for job workers * add some general TL types * add TL types and functions for K2 confdata --- .../memory-resource/resource_allocator.h | 5 +- .../stdlib/crypto/crypto-functions.cpp | 10 +- runtime-light/tl/tl-core.cpp | 2 + runtime-light/tl/tl-core.h | 15 +- runtime-light/tl/tl-functions.cpp | 37 ++- runtime-light/tl/tl-functions.h | 18 +- runtime-light/tl/tl-types.cpp | 39 ++- runtime-light/tl/tl-types.h | 237 +++++++++++++++++- 8 files changed, 330 insertions(+), 33 deletions(-) diff --git a/runtime-core/memory-resource/resource_allocator.h b/runtime-core/memory-resource/resource_allocator.h index eca8f3b1a1..1028040558 100644 --- a/runtime-core/memory-resource/resource_allocator.h +++ b/runtime-core/memory-resource/resource_allocator.h @@ -9,9 +9,9 @@ #include #include #include +#include #include "common/wrappers/likely.h" - #include "runtime-core/utils/kphp-assert-core.h" namespace memory_resource { @@ -79,6 +79,9 @@ using deque = std::deque>; template using queue = std::queue>>; + +template +using vector = std::vector>; } // namespace stl } // namespace memory_resource diff --git a/runtime-light/stdlib/crypto/crypto-functions.cpp b/runtime-light/stdlib/crypto/crypto-functions.cpp index 63e4f32899..aab767c41f 100644 --- a/runtime-light/stdlib/crypto/crypto-functions.cpp +++ b/runtime-light/stdlib/crypto/crypto-functions.cpp @@ -12,7 +12,7 @@ namespace { -const string CRYPTO_COMPONENT = string("crypto"); +constexpr const char *CRYPTO_COMPONENT_NAME = "crypto"; } // namespace @@ -31,7 +31,7 @@ task_t> f$openssl_random_pseudo_bytes(int64_t length) noexcept string request_buf; request_buf.append(buffer.data(), buffer.size()); - auto query = f$component_client_send_request(CRYPTO_COMPONENT, string{buffer.data(), static_cast(buffer.size())}); + auto query = f$component_client_send_request(string{CRYPTO_COMPONENT_NAME}, string{buffer.data(), static_cast(buffer.size())}); string resp = co_await f$component_client_fetch_response(co_await query); buffer.clean(); @@ -55,7 +55,7 @@ task_t>> f$openssl_x509_parse(const string &data, bool sho string request_buf; request_buf.append(buffer.data(), buffer.size()); - auto query = f$component_client_send_request(CRYPTO_COMPONENT, request_buf); + auto query = f$component_client_send_request(string{CRYPTO_COMPONENT_NAME}, request_buf); string resp_from_platform = co_await f$component_client_fetch_response(co_await query); buffer.clean(); @@ -75,7 +75,7 @@ task_t f$openssl_sign(const string &data, string &signature, const string tl::TLBuffer buffer; request.store(buffer); - auto query = f$component_client_send_request(CRYPTO_COMPONENT, string(buffer.data(), buffer.size())); + auto query = f$component_client_send_request(string{CRYPTO_COMPONENT_NAME}, string(buffer.data(), buffer.size())); string resp_from_platform = co_await f$component_client_fetch_response(co_await query); buffer.clean(); @@ -98,7 +98,7 @@ task_t f$openssl_verify(const string &data, const string &signature, co tl::TLBuffer buffer; request.store(buffer); - auto query = f$component_client_send_request(CRYPTO_COMPONENT, string(buffer.data(), buffer.size())); + auto query = f$component_client_send_request(string{CRYPTO_COMPONENT_NAME}, string(buffer.data(), buffer.size())); string resp_from_platform = co_await f$component_client_fetch_response(co_await query); buffer.clean(); diff --git a/runtime-light/tl/tl-core.cpp b/runtime-light/tl/tl-core.cpp index b9e4cfa07f..38fd872795 100644 --- a/runtime-light/tl/tl-core.cpp +++ b/runtime-light/tl/tl-core.cpp @@ -5,6 +5,7 @@ #include "runtime-light/tl/tl-core.h" namespace tl { + void TLBuffer::store_string(std::string_view str) noexcept { const char *str_buf{str.data()}; size_t str_len{str.size()}; @@ -94,4 +95,5 @@ std::string_view TLBuffer::fetch_string() noexcept { adjust(total_len_with_padding - size_len); return response; } + } // namespace tl diff --git a/runtime-light/tl/tl-core.h b/runtime-light/tl/tl-core.h index 68467eff10..660b6e0748 100644 --- a/runtime-light/tl/tl-core.h +++ b/runtime-light/tl/tl-core.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include #include @@ -26,7 +27,7 @@ inline constexpr uint64_t MEDIUM_STRING_MAX_LEN = (static_cast(1) << 2 inline constexpr uint8_t LARGE_STRING_MAGIC = 0xff; inline constexpr uint8_t MEDIUM_STRING_MAGIC = 0xfe; -class TLBuffer : private vk::not_copyable { +class TLBuffer final : private vk::not_copyable { string_buffer m_buffer; size_t m_pos{0}; size_t m_remaining{0}; @@ -57,7 +58,7 @@ class TLBuffer : private vk::not_copyable { } void reset(size_t pos) noexcept { - php_assert(pos >= 0 && pos <= size()); + php_assert(pos <= size()); m_pos = pos; m_remaining = size() - m_pos; } @@ -97,4 +98,14 @@ class TLBuffer : private vk::not_copyable { std::string_view fetch_string() noexcept; }; +template +concept tl_serializable = std::default_initializable && requires(T t, TLBuffer tlb) { + { t.store(tlb) } noexcept -> std::same_as; +}; + +template +concept tl_deserializable = std::default_initializable && requires(T t, TLBuffer tlb) { + { t.fetch(tlb) } noexcept -> std::convertible_to; +}; + } // namespace tl diff --git a/runtime-light/tl/tl-functions.cpp b/runtime-light/tl/tl-functions.cpp index e9f4c2f4d4..e62c07060e 100644 --- a/runtime-light/tl/tl-functions.cpp +++ b/runtime-light/tl/tl-functions.cpp @@ -4,6 +4,7 @@ #include "runtime-light/tl/tl-functions.h" +#include #include #include @@ -12,9 +13,6 @@ namespace { -// magic + flags + image_id + job_id + minimum string size length -constexpr auto K2_INVOKE_JW_MIN_SIZE = sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint64_t) + sizeof(int64_t) + sizeof(uint64_t) + tl::SMALL_STRING_SIZE_LEN; - constexpr auto K2_JOB_WORKER_IGNORE_ANSWER_FLAG = static_cast(1U << 0U); } // namespace @@ -22,17 +20,24 @@ constexpr auto K2_JOB_WORKER_IGNORE_ANSWER_FLAG = static_cast(1U << 0U namespace tl { bool K2InvokeJobWorker::fetch(TLBuffer &tlb) noexcept { - if (tlb.size() < K2_INVOKE_JW_MIN_SIZE || *tlb.fetch_trivial() != K2_INVOKE_JOB_WORKER_MAGIC) { + if (tlb.fetch_trivial().value_or(TL_ZERO) != K2_INVOKE_JOB_WORKER_MAGIC) { + return false; + } + + const auto opt_flags{tlb.fetch_trivial()}; + const auto opt_image_id{tlb.fetch_trivial()}; + const auto opt_job_id{tlb.fetch_trivial()}; + const auto opt_timeout_ns{tlb.fetch_trivial()}; + if (!(opt_flags.has_value() && opt_image_id.has_value() && opt_job_id.has_value() && opt_timeout_ns.has_value())) { return false; } + const auto body_view{tlb.fetch_string()}; - const auto flags{*tlb.fetch_trivial()}; - ignore_answer = static_cast(flags & K2_JOB_WORKER_IGNORE_ANSWER_FLAG); - image_id = *tlb.fetch_trivial(); - job_id = *tlb.fetch_trivial(); - timeout_ns = *tlb.fetch_trivial(); - const std::string_view body_view{tlb.fetch_string()}; - body = string{body_view.data(), static_cast(body_view.size())}; + ignore_answer = static_cast(*opt_flags & K2_JOB_WORKER_IGNORE_ANSWER_FLAG); + image_id = *opt_image_id; + job_id = *opt_job_id; + timeout_ns = *opt_timeout_ns; + body = {body_view.data(), static_cast(body_view.size())}; return true; } @@ -72,4 +77,14 @@ void DigestVerify::store(TLBuffer &tlb) const noexcept { tlb.store_string(std::string_view{signature.c_str(), signature.size()}); } +void ConfdataGet::store(TLBuffer &tlb) const noexcept { + tlb.store_trivial(CONFDATA_GET_MAGIC); + tlb.store_string({key.c_str(), static_cast(key.size())}); +} + +void ConfdataGetWildcard::store(TLBuffer &tlb) const noexcept { + tlb.store_trivial(CONFDATA_GET_WILDCARD_MAGIC); + tlb.store_string({wildcard.c_str(), static_cast(wildcard.size())}); +} + } // namespace tl diff --git a/runtime-light/tl/tl-functions.h b/runtime-light/tl/tl-functions.h index cbadfdae60..aca70c65e3 100644 --- a/runtime-light/tl/tl-functions.h +++ b/runtime-light/tl/tl-functions.h @@ -36,7 +36,6 @@ inline constexpr uint32_t GET_PEM_CERT_INFO_MAGIC = 0xa50c'fd6c; inline constexpr uint32_t DIGEST_SIGN_MAGIC = 0xd345'f658; inline constexpr uint32_t DIGEST_VERIFY_MAGIC = 0x5760'bd0e; - struct GetCryptosecurePseudorandomBytes final { int32_t size{}; @@ -67,4 +66,21 @@ struct DigestVerify final { void store(TLBuffer &tlb) const noexcept; }; +// ===== CONFDATA ===== + +inline constexpr uint32_t CONFDATA_GET_MAGIC = 0xf0eb'cd89; +inline constexpr uint32_t CONFDATA_GET_WILDCARD_MAGIC = 0x5759'bd9e; + +struct ConfdataGet final { + string key; + + void store(TLBuffer &tlb) const noexcept; +}; + +struct ConfdataGetWildcard final { + string wildcard; + + void store(TLBuffer &tlb) const noexcept; +}; + } // namespace tl diff --git a/runtime-light/tl/tl-types.cpp b/runtime-light/tl/tl-types.cpp index 3919cda756..90427ed0db 100644 --- a/runtime-light/tl/tl-types.cpp +++ b/runtime-light/tl/tl-types.cpp @@ -4,6 +4,7 @@ #include "runtime-light/tl/tl-types.h" +#include #include #include #include @@ -13,27 +14,28 @@ namespace { -// magic + flags + job_id + minimum string size length -constexpr auto K2_JOB_WORKER_RESPONSE_MIN_SIZE = sizeof(uint32_t) + sizeof(uint32_t) + sizeof(int64_t) + tl::SMALL_STRING_SIZE_LEN; +enum CertInfoItem : uint32_t { LONG_MAGIC = 0x533f'f89f, STR_MAGIC = 0xc427'feef, DICT_MAGIC = 0x1ea8'a774 }; -enum CertInfoItem : uint32_t { - LONG_MAGIC = 0x533f'f89f, - STR_MAGIC = 0xc427'feef, - DICT_MAGIC = 0x1ea8'a774 -}; +constexpr uint32_t K2_JOB_WORKER_RESPONSE_MAGIC = 0x3afb'3a08; +constexpr uint32_t CONFDATA_VALUE_MAGIC = 0x3eaa'910b; } // namespace namespace tl { bool K2JobWorkerResponse::fetch(TLBuffer &tlb) noexcept { - if (tlb.size() < K2_JOB_WORKER_RESPONSE_MIN_SIZE || *tlb.fetch_trivial() != K2_JOB_WORKER_RESPONSE_MAGIC) { + if (tlb.fetch_trivial().value_or(TL_ZERO) != K2_JOB_WORKER_RESPONSE_MAGIC) { return false; } std::ignore = tlb.fetch_trivial(); // ignore flags - job_id = *tlb.fetch_trivial(); - const std::string_view body_view{tlb.fetch_string()}; + const auto opt_job_id{tlb.fetch_trivial()}; + if (!opt_job_id.has_value()) { + return false; + } + const auto body_view{tlb.fetch_string()}; + + job_id = *opt_job_id; body = string{body_view.data(), static_cast(body_view.size())}; return true; } @@ -130,4 +132,21 @@ bool GetPemCertInfoResponse::fetch(TLBuffer &tlb) noexcept { return true; } +bool ConfdataValue::fetch(TLBuffer &tlb) noexcept { + if (tlb.fetch_trivial().value_or(TL_ZERO) != CONFDATA_VALUE_MAGIC) { + return false; + } + const auto value_view{tlb.fetch_string()}; + Bool is_php_serialized_{}; + Bool is_json_serialized_{}; + if (!(is_php_serialized_.fetch(tlb) && is_json_serialized_.fetch(tlb))) { + return false; + } + + value = {value_view.data(), static_cast(value_view.size())}; + is_php_serialized = is_php_serialized_.value; + is_json_serialized = is_json_serialized_.value; + return true; +} + } // namespace tl diff --git a/runtime-light/tl/tl-types.h b/runtime-light/tl/tl-types.h index d3778eff50..4f75d3d0ad 100644 --- a/runtime-light/tl/tl-types.h +++ b/runtime-light/tl/tl-types.h @@ -4,16 +4,237 @@ #pragma once +#include +#include #include +#include +#include "common/tl/constants/common.h" +#include "runtime-core/allocator/runtime-allocator.h" +#include "runtime-core/memory-resource/resource_allocator.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" #include "runtime-core/runtime-core.h" #include "runtime-light/tl/tl-core.h" namespace tl { -// ===== JOB WORKERS ===== +// ===== GENERAL ===== + +struct Bool final { + bool value{}; + + bool fetch(TLBuffer &tlb) noexcept { + const auto magic{tlb.fetch_trivial().value_or(TL_ZERO)}; + value = magic == TL_BOOL_TRUE; + return magic == TL_BOOL_TRUE || magic == TL_BOOL_FALSE; + } + + void store(TLBuffer &tlb) const noexcept { + tlb.store_trivial(value ? TL_BOOL_TRUE : TL_BOOL_FALSE); + } +}; + +template +struct Maybe final { + std::optional opt_value{}; + + bool fetch(TLBuffer &tlb) noexcept requires tl_deserializable { + const auto magic{tlb.fetch_trivial().value_or(TL_ZERO)}; + if (magic == TL_MAYBE_TRUE) { + opt_value.emplace(); + return (*opt_value).fetch(tlb); + } else if (magic == TL_MAYBE_FALSE) { + opt_value = std::nullopt; + return true; + } + return false; + } + + void store(TLBuffer &tlb) const noexcept requires tl_serializable { + if (opt_value.has_value()) { + tlb.store_trivial(TL_MAYBE_TRUE); + (*opt_value).store(tlb); + } else { + tlb.store_trivial(TL_MAYBE_FALSE); + } + } +}; + +template +struct vector final { + using vector_t = memory_resource::stl::vector; + vector_t data{typename vector_t::allocator_type(RuntimeAllocator::current().memory_resource)}; + + using iterator = vector_t::iterator; + using const_iterator = vector_t::const_iterator; + + iterator begin() noexcept { + return data.begin(); + } + iterator end() noexcept { + return data.end(); + } + const_iterator begin() const noexcept { + return data.begin(); + } + const_iterator end() const noexcept { + return data.end(); + } + const_iterator cbegin() const noexcept { + return data.cbegin(); + } + const_iterator cend() const noexcept { + return data.cend(); + } + + size_t size() const noexcept { + return data.size(); + } + + bool fetch(TLBuffer &tlb) noexcept requires tl_deserializable { + int64_t size{}; + if (const auto opt_size{tlb.fetch_trivial()}; opt_size.has_value()) { + size = *opt_size; + } else { + return false; + } + + data.clear(); + data.reserve(static_cast(size)); + for (auto i = 0; i < size; ++i) { + T t{}; + if (!t.fetch(tlb)) { + return false; + } + data.emplace_back(std::move(t)); + } + + return true; + } + + void store(TLBuffer &tlb) const noexcept requires tl_serializable { + tlb.store_trivial(static_cast(data.size())); + std::for_each(data.cbegin(), data.cend(), [&tlb](auto &&elem) { std::forward(elem).store(tlb); }); + } +}; + +template +struct Vector final { + vector data{}; + + using iterator = vector::iterator; + using const_iterator = vector::const_iterator; + + iterator begin() noexcept { + return data.begin(); + } + iterator end() noexcept { + return data.end(); + } + const_iterator begin() const noexcept { + return data.begin(); + } + const_iterator end() const noexcept { + return data.end(); + } + const_iterator cbegin() const noexcept { + return data.cbegin(); + } + const_iterator cend() const noexcept { + return data.cend(); + } + + size_t size() const noexcept { + return data.size(); + } + + bool fetch(TLBuffer &tlb) noexcept requires tl_deserializable { + if (tlb.fetch_trivial().value_or(TL_ZERO) != TL_VECTOR) { + return false; + } + return data.fetch(tlb); + } -inline constexpr uint32_t K2_JOB_WORKER_RESPONSE_MAGIC = 0x3afb'3a08; + void store(TLBuffer &tlb) const noexcept requires tl_serializable { + tlb.store_trivial(TL_VECTOR); + data.store(tlb); + } +}; + +template +struct dictionaryField final { + string key; + T value{}; + + bool fetch(TLBuffer &tlb) noexcept requires tl_deserializable { + const auto key_view{tlb.fetch_string()}; + key = {key_view.data(), static_cast(key_view.size())}; + return !value.fetch(tlb); + } + + void store(TLBuffer &tlb) const noexcept requires tl_serializable { + tlb.store_string({key.c_str(), static_cast(key.size())}); + value.store(tlb); + } +}; + +template +struct dictionary final { + vector> data{}; + + using iterator = vector>::iterator; + using const_iterator = vector>::const_iterator; + + iterator begin() noexcept { + return data.begin(); + } + iterator end() noexcept { + return data.end(); + } + const_iterator begin() const noexcept { + return data.begin(); + } + const_iterator end() const noexcept { + return data.end(); + } + const_iterator cbegin() const noexcept { + return data.cbegin(); + } + const_iterator cend() const noexcept { + return data.cend(); + } + + size_t size() const noexcept { + return data.size(); + } + + bool fetch(TLBuffer &tlb) noexcept requires tl_deserializable { + return data.fetch(tlb); + } + + void store(TLBuffer &tlb) const noexcept requires tl_serializable { + data.store(tlb); + } +}; + +template +struct Dictionary final { + dictionary data{}; + + bool fetch(TLBuffer &tlb) noexcept requires tl_deserializable { + if (tlb.fetch_trivial().value_or(TL_ZERO) != TL_DICTIONARY) { + return false; + } + return data.fetch(tlb); + } + + void store(TLBuffer &tlb) const noexcept requires tl_serializable { + tlb.store_trivial(TL_DICTIONARY); + data.store(tlb); + } +}; + +// ===== JOB WORKERS ===== struct K2JobWorkerResponse final { int64_t job_id{}; @@ -28,7 +249,7 @@ struct K2JobWorkerResponse final { // Actually it's "Maybe (Dictionary CertInfoItem)" // But I now want to have this logic separately -struct GetPemCertInfoResponse { +struct GetPemCertInfoResponse final { array data; bool fetch(TLBuffer &tlb) noexcept; @@ -47,4 +268,14 @@ enum DigestAlgorithm : uint32_t { MD2 = 0x5aca'6998, }; +// ===== CONFDATA ===== + +struct ConfdataValue final { + string value; + bool is_php_serialized{}; + bool is_json_serialized{}; + + bool fetch(TLBuffer &tlb) noexcept; +}; + } // namespace tl