diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 513d043dca..f4dd877718 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -438,7 +438,8 @@ struct DNS_Resolve_Data { std::shared_ptr dns_cache; std::string hostname; std::set cached_ips; - unsigned int ttl; + unsigned int ttl = 0; + unsigned int refresh_intv = 0; }; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 3caf199e08..9f18d6c892 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "prometheus/counter.h" #include "MySQL_Protocol.h" #include "MySQL_HostGroups_Manager.h" @@ -4668,6 +4669,13 @@ void* monitor_dns_resolver_thread(void* args) { if (!ips.empty()) { bool to_update_cache = false; + int cache_ttl = dns_resolve_data->ttl; + if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { + thread_local std::mt19937 gen(std::random_device{}()); + const int jitter = static_cast(dns_resolve_data->ttl * 0.025); + std::uniform_int_distribution dis(-jitter, jitter); + cache_ttl += dis(gen); + } if (!dns_resolve_data->cached_ips.empty()) { @@ -4686,14 +4694,14 @@ void* monitor_dns_resolver_thread(void* args) { // only update dns_records_bookkeeping if (!to_update_cache) { proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str()); - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl)))); } } else to_update_cache = true; if (to_update_cache) { - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl)))); dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); } @@ -4841,7 +4849,18 @@ void* MySQL_Monitor::monitor_dns_cache() { std::list>> dns_resolve_result; + int delay_us = 100; + if (hostnames.empty() == false) { + delay_us = mysql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size(); + delay_us *= 40; + if (delay_us > 1000000 || delay_us <= 0) { + delay_us = 10000; + } + delay_us = delay_us + rand() % delay_us; + } + if (dns_records_bookkeeping.empty() == false) { + unsigned long long current_time = monotonic_time(); for (auto itr = dns_records_bookkeeping.begin(); @@ -4861,12 +4880,14 @@ void* MySQL_Monitor::monitor_dns_cache() { dns_resolve_data->hostname = std::move(itr->hostname_); dns_resolve_data->cached_ips = std::move(itr->ips_); dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); itr = dns_records_bookkeeping.erase(itr); + usleep(delay_us); continue; } @@ -4881,7 +4902,6 @@ void* MySQL_Monitor::monitor_dns_cache() { if (qsize > (static_cast(mysql_thread___monitor_local_dns_resolver_queue_maxsize) / 8)) { proxy_warning("DNS resolver queue too big: %d. Please refer to https://proxysql.com/documentation/dns-cache/ for further information.\n", qsize); - unsigned int threads_max = num_dns_resolver_max_threads; if (threads_max > num_threads) { @@ -4906,14 +4926,15 @@ void* MySQL_Monitor::monitor_dns_cache() { } if (hostnames.empty() == false) { - for (const std::string& hostname : hostnames) { std::unique_ptr dns_resolve_data(new DNS_Resolve_Data()); dns_resolve_data->hostname = hostname; dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); + usleep(delay_us); } } diff --git a/test/tap/tests/test_dns_cache-t.cpp b/test/tap/tests/test_dns_cache-t.cpp index 7e14bfa0d2..9ea4d9de90 100644 --- a/test/tap/tests/test_dns_cache-t.cpp +++ b/test/tap/tests/test_dns_cache-t.cpp @@ -109,7 +109,10 @@ bool check_result(const std::string& key, const std::map& p COMPARE fn; bool res = fn(after_metric_val, prev_metric_val); - ok(res, "'%s' metric result success.",key.c_str()); + std::string bin_op_name = typeid(COMPARE).name(); + bin_op_name = bin_op_name.substr(3, bin_op_name.size() - 6); + + ok(res, "'%s' metric result should be '%s' %f. %f",key.c_str(), bin_op_name.c_str(), after_metric_val, prev_metric_val); } return true; @@ -161,14 +164,14 @@ int main(int argc, char** argv) { DECLARE_PREV_AFTER_METRICS(); std::vector>> dns_cache_check_steps = { - STEP_START + STEP_START // Step: 0 EXECUTE_QUERY("SET mysql-monitor_enabled='false'", proxysql_admin, false), - EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=1000", proxysql_admin, false), + EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=500", proxysql_admin, false), EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_ttl=5000", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false), DELAY_SEC(2) STEP_END, - STEP_START + STEP_START // Step: 1 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'0.0.0.0',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -178,7 +181,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 2 UPDATE_PREV_METRICS(proxysql_admin), LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2), DELAY_SEC(2), @@ -187,7 +190,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 3 UPDATE_PREV_METRICS(proxysql_admin), LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2), DELAY_SEC(2), @@ -196,7 +199,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 4 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'google.com',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -208,7 +211,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 5 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,' yahoo.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -220,7 +223,17 @@ int main(int argc, char** argv) { CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 6 + UPDATE_PREV_METRICS(proxysql_admin), + EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'amazon.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false), + EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), + DELAY_SEC(2), + UPDATE_AFTER_METRICS(proxysql_admin), + CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_record_updated"), + CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), + CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") + STEP_END, + STEP_START // Step: 7 UPDATE_PREV_METRICS(proxysql_admin), // EXECUTE_QUERY("DELETE FROM mysql_servers WHERE hostgroup_id=999", proxysql_admin, false), EXECUTE_QUERY("DELETE FROM mysql_servers", proxysql_admin, false), @@ -231,7 +244,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 8 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'INVALID_DOMAIN',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -243,7 +256,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 9 //disable dns cache EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=0", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false),