Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DNS cache enhancement #4729

Open
wants to merge 3 commits into
base: v2.7
Choose a base branch
from
Open

DNS cache enhancement #4729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cfc2c97
* Introduced a 5% jitter to cache_ttl to increase the likelihood of
rahim-kanji Oct 25, 2024
11d8d80
Optimization
rahim-kanji Oct 28, 2024
90b33dc
Updated TAP test
rahim-kanji Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion include/MySQL_Monitor.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,8 @@ struct DNS_Resolve_Data {
std::shared_ptr<DNS_Cache> dns_cache;
std::string hostname;
std::set<std::string> cached_ips;
unsigned int ttl;
unsigned int ttl = 0;
unsigned int refresh_intv = 0;
};


Expand Down
29 changes: 25 additions & 4 deletions lib/MySQL_Monitor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <thread>
#include <future>
#include <sstream>
#include <random>
#include "prometheus/counter.h"
#include "MySQL_Protocol.h"
#include "MySQL_HostGroups_Manager.h"
Expand Down Expand Up @@ -4668,6 +4669,13 @@ void* monitor_dns_resolver_thread(void* args) {
if (!ips.empty()) {

bool to_update_cache = false;
int cache_ttl = dns_resolve_data->ttl;
if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) {
thread_local std::mt19937 gen(std::random_device{}());
const int jitter = static_cast<int>(dns_resolve_data->ttl * 0.025);
std::uniform_int_distribution<int> dis(-jitter, jitter);
cache_ttl += dis(gen);
}

if (!dns_resolve_data->cached_ips.empty()) {

Expand All @@ -4686,14 +4694,14 @@ void* monitor_dns_resolver_thread(void* args) {
// only update dns_records_bookkeeping
if (!to_update_cache) {
proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str());
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * dns_resolve_data->ttl))));
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl))));
}
}
else
to_update_cache = true;

if (to_update_cache) {
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * dns_resolve_data->ttl))));
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl))));
dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips));
}

Expand Down Expand Up @@ -4841,7 +4849,18 @@ void* MySQL_Monitor::monitor_dns_cache() {

std::list<std::future<std::tuple<bool, DNS_Cache_Record>>> dns_resolve_result;

int delay_us = 100;
if (hostnames.empty() == false) {
delay_us = mysql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size();
delay_us *= 40;
if (delay_us > 1000000 || delay_us <= 0) {
delay_us = 10000;
}
delay_us = delay_us + rand() % delay_us;
}

if (dns_records_bookkeeping.empty() == false) {

unsigned long long current_time = monotonic_time();

for (auto itr = dns_records_bookkeeping.begin();
Expand All @@ -4861,12 +4880,14 @@ void* MySQL_Monitor::monitor_dns_cache() {
dns_resolve_data->hostname = std::move(itr->hostname_);
dns_resolve_data->cached_ips = std::move(itr->ips_);
dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl;
dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval;
dns_resolve_data->dns_cache = dns_cache;
dns_resolve_result.emplace_back(dns_resolve_data->result.get_future());

proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str());
dns_resolver_queue.add(new WorkItem<DNS_Resolve_Data>(dns_resolve_data.release(), monitor_dns_resolver_thread));
itr = dns_records_bookkeeping.erase(itr);
usleep(delay_us);
continue;
}

Expand All @@ -4881,7 +4902,6 @@ void* MySQL_Monitor::monitor_dns_cache() {

if (qsize > (static_cast<unsigned int>(mysql_thread___monitor_local_dns_resolver_queue_maxsize) / 8)) {
proxy_warning("DNS resolver queue too big: %d. Please refer to https://proxysql.com/documentation/dns-cache/ for further information.\n", qsize);

unsigned int threads_max = num_dns_resolver_max_threads;

if (threads_max > num_threads) {
Expand All @@ -4906,14 +4926,15 @@ void* MySQL_Monitor::monitor_dns_cache() {
}

if (hostnames.empty() == false) {

for (const std::string& hostname : hostnames) {
std::unique_ptr<DNS_Resolve_Data> dns_resolve_data(new DNS_Resolve_Data());
dns_resolve_data->hostname = hostname;
dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl;
dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval;
dns_resolve_data->dns_cache = dns_cache;
dns_resolve_result.emplace_back(dns_resolve_data->result.get_future());
dns_resolver_queue.add(new WorkItem<DNS_Resolve_Data>(dns_resolve_data.release(), monitor_dns_resolver_thread));
usleep(delay_us);
}
}

Expand Down
35 changes: 24 additions & 11 deletions test/tap/tests/test_dns_cache-t.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,10 @@ bool check_result(const std::string& key, const std::map<std::string, double>& p
COMPARE fn;
bool res = fn(after_metric_val, prev_metric_val);

ok(res, "'%s' metric result success.",key.c_str());
std::string bin_op_name = typeid(COMPARE).name();
bin_op_name = bin_op_name.substr(3, bin_op_name.size() - 6);

ok(res, "'%s' metric result should be '%s' %f. %f",key.c_str(), bin_op_name.c_str(), after_metric_val, prev_metric_val);
}

return true;
Expand Down Expand Up @@ -161,14 +164,14 @@ int main(int argc, char** argv) {
DECLARE_PREV_AFTER_METRICS();

std::vector<std::vector<std::function<bool()>>> dns_cache_check_steps = {
STEP_START
STEP_START // Step: 0
EXECUTE_QUERY("SET mysql-monitor_enabled='false'", proxysql_admin, false),
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=1000", proxysql_admin, false),
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=500", proxysql_admin, false),
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_ttl=5000", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false),
DELAY_SEC(2)
STEP_END,
STEP_START
STEP_START // Step: 1
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'0.0.0.0',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -178,7 +181,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 2
UPDATE_PREV_METRICS(proxysql_admin),
LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2),
DELAY_SEC(2),
Expand All @@ -187,7 +190,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 3
UPDATE_PREV_METRICS(proxysql_admin),
LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2),
DELAY_SEC(2),
Expand All @@ -196,7 +199,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 4
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'google.com',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -208,7 +211,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 5
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,' yahoo.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -220,7 +223,17 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 6
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'amazon.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
DELAY_SEC(2),
UPDATE_AFTER_METRICS(proxysql_admin),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_record_updated"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START // Step: 7
UPDATE_PREV_METRICS(proxysql_admin),
// EXECUTE_QUERY("DELETE FROM mysql_servers WHERE hostgroup_id=999", proxysql_admin, false),
EXECUTE_QUERY("DELETE FROM mysql_servers", proxysql_admin, false),
Expand All @@ -231,7 +244,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 8
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'INVALID_DOMAIN',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -243,7 +256,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 9
//disable dns cache
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=0", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false),
Expand Down