diff --git a/server/php-queries.h b/server/php-queries.h index 868c37ca48..a49cf4a87f 100644 --- a/server/php-queries.h +++ b/server/php-queries.h @@ -77,6 +77,24 @@ struct net_query_t { std::variant> data; }; +namespace slow_net_event_stats { + +struct slow_rpc_response_stats final { + const char *tl_function_name{nullptr}; + int32_t actor_or_port{}; + double response_time{}; + bool is_error{}; +}; + +struct slow_job_worker_response_stats final { + const char *class_name{nullptr}; + double response_time{}; +}; + +using stats_t = std::variant; + +}; // namespace slow_net_event_stats + #pragma pack(push, 4) /*** diff --git a/server/php-runner.cpp b/server/php-runner.cpp index 915ad59949..d02d04d3ef 100644 --- a/server/php-runner.cpp +++ b/server/php-runner.cpp @@ -4,38 +4,40 @@ #include "server/php-runner.h" -#include #include #include #include #include -#include -#include #include #include #include +#include #include "common/fast-backtrace.h" #include "common/kernel-version.h" #include "common/kprintf.h" +#include "common/precise-time.h" #include "common/wrappers/memory-utils.h" -#include "net/net-connections.h" - +#include "common/wrappers/overloaded.h" #include "runtime/allocator.h" #include "runtime/critical_section.h" #include "runtime/curl.h" #include "runtime/exception.h" #include "runtime/interface.h" +#include "runtime/job-workers/processing-jobs.h" #include "runtime/kphp_tracing.h" #include "runtime/oom_handler.h" -#include "runtime/profiler.h" #include "runtime/php_assert.h" +#include "runtime/profiler.h" +#include "runtime/rpc.h" +#include "runtime/tl/tl_magics_decoding.h" #include "server/json-logger.h" #include "server/php-engine-vars.h" #include "server/php-queries.h" #include "server/server-log.h" #include "server/server-stats.h" #include "server/signal-handlers.h" +#include "server/statshouse/statshouse-manager.h" DEFINE_VERBOSITY(php_runner); @@ -50,6 +52,32 @@ namespace { [[maybe_unused]] const void *main_thread_stack = nullptr; [[maybe_unused]] size_t main_thread_stacksize = 0; + +void send_slow_net_event_stats(const net_event_t &event, double time_sec) noexcept { + std::visit(overloaded{ + [&event, time_sec](const net_events_data::rpc_answer &) noexcept { + const auto *rpc_req = get_rpc_request(event.slot_id); + StatsHouseManager::get().add_slow_net_event_stats( + slow_net_event_stats::slow_rpc_response_stats{tl_magic_convert_to_name(rpc_req->function_magic), rpc_req->actor_or_port, time_sec, false}); + }, + [&event, time_sec](const net_events_data::rpc_error &) noexcept { + const auto *rpc_req = get_rpc_request(event.slot_id); + StatsHouseManager::get().add_slow_net_event_stats( + slow_net_event_stats::slow_rpc_response_stats{tl_magic_convert_to_name(rpc_req->function_magic), rpc_req->actor_or_port, time_sec, true}); + }, + [time_sec](const net_events_data::job_worker_answer &jw_answer) noexcept { + if (jw_answer.job_result != nullptr) { + StatsHouseManager::get().add_slow_net_event_stats( + slow_net_event_stats::slow_job_worker_response_stats{jw_answer.job_result->response.get_class(), time_sec}); + } + }, + [](const database_drivers::Response *) {}, + [](const curl_async::CurlResponse *) {}, + + }, + event.data); +} + } // namespace void PhpScript::error(const char *error_message, script_error_t error_type) noexcept { @@ -271,6 +299,7 @@ void PhpScript::update_net_time() noexcept { ++long_queries_cnt; kprintf("LONG query: %lf\n", net_add); if (const net_event_t *event = get_last_net_event()) { + send_slow_net_event_stats(*event, net_add); kprintf("Awakening net event: %s\n", event->get_description()); } } diff --git a/server/statshouse/statshouse-manager.cpp b/server/statshouse/statshouse-manager.cpp index da0c88b6c7..d97afbfd7a 100644 --- a/server/statshouse/statshouse-manager.cpp +++ b/server/statshouse/statshouse-manager.cpp @@ -4,15 +4,21 @@ #include "server/statshouse/statshouse-manager.h" +#include +#include #include +#include #include +#include #include "common/precise-time.h" #include "common/resolver.h" +#include "common/wrappers/overloaded.h" #include "runtime/instance-cache.h" -#include "server/job-workers/shared-memory-manager.h" #include "server/confdata-stats.h" +#include "server/job-workers/shared-memory-manager.h" #include "server/json-logger.h" +#include "server/php-queries.h" #include "server/php-runner.h" #include "server/server-config.h" #include "server/server-stats.h" @@ -333,3 +339,23 @@ void StatsHouseManager::add_confdata_master_stats(const ConfdataStats &confdata_ } } } + +void StatsHouseManager::add_slow_net_event_stats(const slow_net_event_stats::stats_t &stats) noexcept { + std::visit(overloaded{[this](const slow_net_event_stats::slow_rpc_response_stats &rpc_query_stat) noexcept { + // FIXME: it's enough to have it equal 10, but due to bug in GCC we are forced to use a length > 253 + constexpr auto MAX_INT_STRING_LENGTH = 254; + std::array buf{}; + const auto chars{std::to_chars(buf.data(), buf.data() + buf.size(), rpc_query_stat.actor_or_port)}; + client.metric("kphp_slow_rpc_response") + .tag(rpc_query_stat.tl_function_name != nullptr ? rpc_query_stat.tl_function_name : "unknown") + .tag({buf.data(), static_cast(chars.ptr - buf.data())}) + .tag(rpc_query_stat.is_error ? "error" : "success") + .write_value(rpc_query_stat.response_time); + }, + [this](const slow_net_event_stats::slow_job_worker_response_stats &jw_response_stat) noexcept { + client.metric("kphp_slow_job_worker_response") + .tag(jw_response_stat.class_name != nullptr ? jw_response_stat.class_name : "unknown") + .write_value(jw_response_stat.response_time); + }}, + stats); +} diff --git a/server/statshouse/statshouse-manager.h b/server/statshouse/statshouse-manager.h index fd8a552710..31f64a3e7f 100644 --- a/server/statshouse/statshouse-manager.h +++ b/server/statshouse/statshouse-manager.h @@ -11,6 +11,7 @@ #include "common/mixin/not_copyable.h" #include "runtime-core/memory-resource/memory_resource.h" #include "server/job-workers/job-stats.h" +#include "server/php-queries.h" #include "server/statshouse/statshouse-client.h" #include "server/workers-control.h" #include "server/workers-stats.h" @@ -88,6 +89,8 @@ class StatsHouseManager : vk::not_copyable { void add_confdata_master_stats(const ConfdataStats &confdata_stats); + void add_slow_net_event_stats(const slow_net_event_stats::stats_t &stats) noexcept; + private: StatsHouseClient client; bool need_write_enable_tag_host = false;