From cfc2c977a7a18151d554289afd8fd1a2e570e117 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Fri, 25 Oct 2024 15:44:13 +0500 Subject: [PATCH 1/3] * Introduced a 5% jitter to cache_ttl to increase the likelihood of distributing DNS resolution across multiple intervals, reducing the chance of simultaneous expiry times. * Added a small delay after queuing DNS records, allowing DNS resolver thread time to process as many hostnames as possible before assessing the need for additional helper threads. This helps prevent 'DNS resolver queue too big' warnings in the logs. --- include/MySQL_Monitor.hpp | 3 ++- lib/MySQL_Monitor.cpp | 30 ++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 513d043dc..f4dd87771 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -438,7 +438,8 @@ struct DNS_Resolve_Data { std::shared_ptr dns_cache; std::string hostname; std::set cached_ips; - unsigned int ttl; + unsigned int ttl = 0; + unsigned int refresh_intv = 0; }; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 3caf199e0..a776aa595 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "prometheus/counter.h" #include "MySQL_Protocol.h" #include "MySQL_HostGroups_Manager.h" @@ -4668,6 +4669,14 @@ void* monitor_dns_resolver_thread(void* args) { if (!ips.empty()) { bool to_update_cache = false; + int cache_ttl = dns_resolve_data->ttl; + if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { + std::random_device rd; + std::mt19937 gen(rd()); + const int jitter = static_cast(dns_resolve_data->ttl * 0.025); + std::uniform_int_distribution dis(-jitter, jitter); + cache_ttl += dis(gen); + } if (!dns_resolve_data->cached_ips.empty()) { @@ -4686,14 +4695,14 @@ void* monitor_dns_resolver_thread(void* args) { // only update dns_records_bookkeeping if (!to_update_cache) { proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str()); - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl)))); } } else to_update_cache = true; if (to_update_cache) { - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl)))); dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); } @@ -4841,7 +4850,18 @@ void* MySQL_Monitor::monitor_dns_cache() { std::list>> dns_resolve_result; + int delay_us = 100; + if (hostnames.empty() == false) { + delay_us = mysql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size(); + delay_us *= 40; + if (delay_us > 1000000 || delay_us <= 0) { + delay_us = 10000; + } + delay_us = delay_us + rand() % delay_us; + } + if (dns_records_bookkeeping.empty() == false) { + unsigned long long current_time = monotonic_time(); for (auto itr = dns_records_bookkeeping.begin(); @@ -4861,12 +4881,14 @@ void* MySQL_Monitor::monitor_dns_cache() { dns_resolve_data->hostname = std::move(itr->hostname_); dns_resolve_data->cached_ips = std::move(itr->ips_); dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); itr = dns_records_bookkeeping.erase(itr); + usleep(delay_us); continue; } @@ -4881,7 +4903,6 @@ void* MySQL_Monitor::monitor_dns_cache() { if (qsize > (static_cast(mysql_thread___monitor_local_dns_resolver_queue_maxsize) / 8)) { proxy_warning("DNS resolver queue too big: %d. Please refer to https://proxysql.com/documentation/dns-cache/ for further information.\n", qsize); - unsigned int threads_max = num_dns_resolver_max_threads; if (threads_max > num_threads) { @@ -4906,14 +4927,15 @@ void* MySQL_Monitor::monitor_dns_cache() { } if (hostnames.empty() == false) { - for (const std::string& hostname : hostnames) { std::unique_ptr dns_resolve_data(new DNS_Resolve_Data()); dns_resolve_data->hostname = hostname; dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); + usleep(delay_us); } } From 11d8d805201ebd11126937bca86379a7859ea382 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Mon, 28 Oct 2024 11:32:41 +0500 Subject: [PATCH 2/3] Optimization --- lib/MySQL_Monitor.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a776aa595..9f18d6c89 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -4671,8 +4671,7 @@ void* monitor_dns_resolver_thread(void* args) { bool to_update_cache = false; int cache_ttl = dns_resolve_data->ttl; if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { - std::random_device rd; - std::mt19937 gen(rd()); + thread_local std::mt19937 gen(std::random_device{}()); const int jitter = static_cast(dns_resolve_data->ttl * 0.025); std::uniform_int_distribution dis(-jitter, jitter); cache_ttl += dis(gen); From 90b33dce31281554c871cae61677e3d74de627f7 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Tue, 29 Oct 2024 12:09:38 +0500 Subject: [PATCH 3/3] Updated TAP test --- test/tap/tests/test_dns_cache-t.cpp | 35 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/test/tap/tests/test_dns_cache-t.cpp b/test/tap/tests/test_dns_cache-t.cpp index 7e14bfa0d..9ea4d9de9 100644 --- a/test/tap/tests/test_dns_cache-t.cpp +++ b/test/tap/tests/test_dns_cache-t.cpp @@ -109,7 +109,10 @@ bool check_result(const std::string& key, const std::map& p COMPARE fn; bool res = fn(after_metric_val, prev_metric_val); - ok(res, "'%s' metric result success.",key.c_str()); + std::string bin_op_name = typeid(COMPARE).name(); + bin_op_name = bin_op_name.substr(3, bin_op_name.size() - 6); + + ok(res, "'%s' metric result should be '%s' %f. %f",key.c_str(), bin_op_name.c_str(), after_metric_val, prev_metric_val); } return true; @@ -161,14 +164,14 @@ int main(int argc, char** argv) { DECLARE_PREV_AFTER_METRICS(); std::vector>> dns_cache_check_steps = { - STEP_START + STEP_START // Step: 0 EXECUTE_QUERY("SET mysql-monitor_enabled='false'", proxysql_admin, false), - EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=1000", proxysql_admin, false), + EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=500", proxysql_admin, false), EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_ttl=5000", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false), DELAY_SEC(2) STEP_END, - STEP_START + STEP_START // Step: 1 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'0.0.0.0',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -178,7 +181,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 2 UPDATE_PREV_METRICS(proxysql_admin), LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2), DELAY_SEC(2), @@ -187,7 +190,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 3 UPDATE_PREV_METRICS(proxysql_admin), LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2), DELAY_SEC(2), @@ -196,7 +199,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 4 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'google.com',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -208,7 +211,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 5 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,' yahoo.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -220,7 +223,17 @@ int main(int argc, char** argv) { CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 6 + UPDATE_PREV_METRICS(proxysql_admin), + EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'amazon.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false), + EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), + DELAY_SEC(2), + UPDATE_AFTER_METRICS(proxysql_admin), + CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_record_updated"), + CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), + CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") + STEP_END, + STEP_START // Step: 7 UPDATE_PREV_METRICS(proxysql_admin), // EXECUTE_QUERY("DELETE FROM mysql_servers WHERE hostgroup_id=999", proxysql_admin, false), EXECUTE_QUERY("DELETE FROM mysql_servers", proxysql_admin, false), @@ -231,7 +244,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 8 UPDATE_PREV_METRICS(proxysql_admin), EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'INVALID_DOMAIN',7861,0,1000,'dummy mysql server')", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false), @@ -243,7 +256,7 @@ int main(int argc, char** argv) { CHECK_RESULT(std::equal_to, "proxysql_mysql_monitor_dns_cache_lookup_success"), CHECK_RESULT(std::greater, "proxysql_mysql_monitor_dns_cache_queried") STEP_END, - STEP_START + STEP_START // Step: 9 //disable dns cache EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=0", proxysql_admin, false), EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false),