Skip to content

Commit

Permalink
Merge pull request #4729 from sysown/2.7_randomized_cache_ttl
Browse files Browse the repository at this point in the history 
DNS cache enhancement
  • Loading branch information
@renecannao
renecannao authored Nov 11, 2024
2 parents 065ee92 + 90b33dc commit b95f1dd
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 16 deletions.
3 changes: 2 additions & 1 deletion include/MySQL_Monitor.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,8 @@ struct DNS_Resolve_Data {
std::shared_ptr<DNS_Cache> dns_cache;
std::string hostname;
std::set<std::string> cached_ips;
unsigned int ttl;
unsigned int ttl = 0;
unsigned int refresh_intv = 0;
};


Expand Down
29 changes: 25 additions & 4 deletions lib/MySQL_Monitor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <thread>
#include <future>
#include <sstream>
#include <random>
#include "prometheus/counter.h"
#include "MySQL_Protocol.h"
#include "MySQL_HostGroups_Manager.h"
Expand Down Expand Up @@ -4668,6 +4669,13 @@ void* monitor_dns_resolver_thread(void* args) {
if (!ips.empty()) {

bool to_update_cache = false;
int cache_ttl = dns_resolve_data->ttl;
if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) {
thread_local std::mt19937 gen(std::random_device{}());
const int jitter = static_cast<int>(dns_resolve_data->ttl * 0.025);
std::uniform_int_distribution<int> dis(-jitter, jitter);
cache_ttl += dis(gen);
}

if (!dns_resolve_data->cached_ips.empty()) {

Expand All @@ -4686,14 +4694,14 @@ void* monitor_dns_resolver_thread(void* args) {
// only update dns_records_bookkeeping
if (!to_update_cache) {
proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str());
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * dns_resolve_data->ttl))));
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl))));
}
}
else
to_update_cache = true;

if (to_update_cache) {
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * dns_resolve_data->ttl))));
dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl))));
dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips));
}

Expand Down Expand Up @@ -4841,7 +4849,18 @@ void* MySQL_Monitor::monitor_dns_cache() {

std::list<std::future<std::tuple<bool, DNS_Cache_Record>>> dns_resolve_result;

int delay_us = 100;
if (hostnames.empty() == false) {
delay_us = mysql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size();
delay_us *= 40;
if (delay_us > 1000000 || delay_us <= 0) {
delay_us = 10000;
}
delay_us = delay_us + rand() % delay_us;
}

if (dns_records_bookkeeping.empty() == false) {

unsigned long long current_time = monotonic_time();

for (auto itr = dns_records_bookkeeping.begin();
Expand All @@ -4861,12 +4880,14 @@ void* MySQL_Monitor::monitor_dns_cache() {
dns_resolve_data->hostname = std::move(itr->hostname_);
dns_resolve_data->cached_ips = std::move(itr->ips_);
dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl;
dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval;
dns_resolve_data->dns_cache = dns_cache;
dns_resolve_result.emplace_back(dns_resolve_data->result.get_future());

proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str());
dns_resolver_queue.add(new WorkItem<DNS_Resolve_Data>(dns_resolve_data.release(), monitor_dns_resolver_thread));
itr = dns_records_bookkeeping.erase(itr);
usleep(delay_us);
continue;
}

Expand All @@ -4881,7 +4902,6 @@ void* MySQL_Monitor::monitor_dns_cache() {

if (qsize > (static_cast<unsigned int>(mysql_thread___monitor_local_dns_resolver_queue_maxsize) / 8)) {
proxy_warning("DNS resolver queue too big: %d. Please refer to https://proxysql.com/documentation/dns-cache/ for further information.\n", qsize);

unsigned int threads_max = num_dns_resolver_max_threads;

if (threads_max > num_threads) {
Expand All @@ -4906,14 +4926,15 @@ void* MySQL_Monitor::monitor_dns_cache() {
}

if (hostnames.empty() == false) {

for (const std::string& hostname : hostnames) {
std::unique_ptr<DNS_Resolve_Data> dns_resolve_data(new DNS_Resolve_Data());
dns_resolve_data->hostname = hostname;
dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl;
dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval;
dns_resolve_data->dns_cache = dns_cache;
dns_resolve_result.emplace_back(dns_resolve_data->result.get_future());
dns_resolver_queue.add(new WorkItem<DNS_Resolve_Data>(dns_resolve_data.release(), monitor_dns_resolver_thread));
usleep(delay_us);
}
}

Expand Down
35 changes: 24 additions & 11 deletions test/tap/tests/test_dns_cache-t.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,10 @@ bool check_result(const std::string& key, const std::map<std::string, double>& p
COMPARE fn;
bool res = fn(after_metric_val, prev_metric_val);

ok(res, "'%s' metric result success.",key.c_str());
std::string bin_op_name = typeid(COMPARE).name();
bin_op_name = bin_op_name.substr(3, bin_op_name.size() - 6);

ok(res, "'%s' metric result should be '%s' %f. %f",key.c_str(), bin_op_name.c_str(), after_metric_val, prev_metric_val);
}

return true;
Expand Down Expand Up @@ -161,14 +164,14 @@ int main(int argc, char** argv) {
DECLARE_PREV_AFTER_METRICS();

std::vector<std::vector<std::function<bool()>>> dns_cache_check_steps = {
STEP_START
STEP_START // Step: 0
EXECUTE_QUERY("SET mysql-monitor_enabled='false'", proxysql_admin, false),
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=1000", proxysql_admin, false),
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=500", proxysql_admin, false),
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_ttl=5000", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false),
DELAY_SEC(2)
STEP_END,
STEP_START
STEP_START // Step: 1
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'0.0.0.0',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -178,7 +181,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 2
UPDATE_PREV_METRICS(proxysql_admin),
LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2),
DELAY_SEC(2),
Expand All @@ -187,7 +190,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 3
UPDATE_PREV_METRICS(proxysql_admin),
LOOP_FUNC(EXECUTE_QUERY("DO 1", proxysql, true), 2),
DELAY_SEC(2),
Expand All @@ -196,7 +199,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 4
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'google.com',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -208,7 +211,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 5
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,' yahoo.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -220,7 +223,17 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 6
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'amazon.com ',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
DELAY_SEC(2),
UPDATE_AFTER_METRICS(proxysql_admin),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_record_updated"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START // Step: 7
UPDATE_PREV_METRICS(proxysql_admin),
// EXECUTE_QUERY("DELETE FROM mysql_servers WHERE hostgroup_id=999", proxysql_admin, false),
EXECUTE_QUERY("DELETE FROM mysql_servers", proxysql_admin, false),
Expand All @@ -231,7 +244,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 8
UPDATE_PREV_METRICS(proxysql_admin),
EXECUTE_QUERY("INSERT INTO mysql_servers (hostgroup_id,hostname,port,max_replication_lag,max_connections,comment) VALUES (999,'INVALID_DOMAIN',7861,0,1000,'dummy mysql server')", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL SERVERS TO RUNTIME", proxysql_admin, false),
Expand All @@ -243,7 +256,7 @@ int main(int argc, char** argv) {
CHECK_RESULT(std::equal_to<double>, "proxysql_mysql_monitor_dns_cache_lookup_success"),
CHECK_RESULT(std::greater<double>, "proxysql_mysql_monitor_dns_cache_queried")
STEP_END,
STEP_START
STEP_START // Step: 9
//disable dns cache
EXECUTE_QUERY("SET mysql-monitor_local_dns_cache_refresh_interval=0", proxysql_admin, false),
EXECUTE_QUERY("LOAD MYSQL VARIABLES TO RUNTIME", proxysql_admin, false),
Expand Down

0 comments on commit b95f1dd

Please sign in to comment.