From 12d830cb35ece8c7abfa5b5dd4b4b85786b7fb3f Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Thu, 6 Apr 2023 11:55:14 +0500 Subject: [PATCH 01/56] Refactoring ProxySQL Cluster * Added separate fetching of mysql_server_incoming and runtime_mysql_server records based on algorithm selection. * Few memory leaks fix * Code refactoring --- include/MySQL_HostGroups_Manager.h | 27 +- include/ProxySQL_Cluster.hpp | 71 ++- include/proxysql_admin.h | 32 +- include/proxysql_glovars.hpp | 1 + include/proxysql_structs.h | 1 + lib/MySQL_HostGroups_Manager.cpp | 843 +++++++++++++++++++++-------- lib/ProxySQL_Admin.cpp | 75 ++- lib/ProxySQL_Cluster.cpp | 729 +++++++++++++++++++++---- lib/ProxySQL_GloVars.cpp | 5 + lib/debug.cpp | 1 + 10 files changed, 1414 insertions(+), 371 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 00ec96778e..332fd161d6 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -66,6 +66,7 @@ using json = nlohmann::json; #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" +#define MYHGM_GEN_INCOMING_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming ORDER BY hostgroup_id, hostname, port" typedef std::unordered_map umap_mysql_errors; @@ -399,6 +400,7 @@ class MySQL_HostGroups_Manager { MYSQL_AWS_AURORA_HOSTGROUPS, MYSQL_HOSTGROUP_ATTRIBUTES, + MYSQL_SERVERS_INCOMING, __HGM_TABLES_SIZE }; @@ -500,7 +502,7 @@ class MySQL_HostGroups_Manager { /** * @brief This resultset holds the current values for 'runtime_mysql_servers' computed by either latest * 'commit' or fetched from another Cluster node. It's also used by ProxySQL_Admin to respond to the - * intercepted query 'CLUSTER_QUERY_MYSQL_SERVERS'. + * intercepted query 'CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS'. * @details This resultset can't right now just contain the value for 'incoming_mysql_servers' as with the * rest of the intercepted resultset. This is due to 'runtime_mysql_servers' reconfigurations that can be * triggered by monitoring actions like 'Galera' currently performs. These actions not only trigger status @@ -551,6 +553,8 @@ class MySQL_HostGroups_Manager { void generate_mysql_hostgroup_attributes_table(); SQLite3_result *incoming_hostgroup_attributes; + SQLite3_result* incoming_mysql_servers; + std::thread *HGCU_thread; std::thread *GTID_syncer_thread; @@ -694,9 +698,11 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); - bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0); - void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init); - void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() + void update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); + bool commit(SQLite3_result* runtime_mysql_servers = nullptr, SQLite3_result* mysql_servers_incoming = nullptr, + const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming = {}); + void commit_update_checksums_from_tables(); + void CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() /** * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime. @@ -704,6 +710,14 @@ class MySQL_HostGroups_Manager { * @param The resulset to be stored replacing the current one. */ void save_runtime_mysql_servers(SQLite3_result *); + + /** + * @brief Store the resultset for the 'mysql_servers_incoming' table. + * The store configuration is later used by Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + void save_mysql_servers_incoming(SQLite3_result* s); + /** * @brief These setters/getter functions store and retrieve the currently hold resultset for the * 'incoming_*' table set that have been loaded to runtime. The store configuration is later used by @@ -810,6 +824,11 @@ class MySQL_HostGroups_Manager { void shutdown(); void unshun_server_all_hostgroups(const char * address, uint16_t port, time_t t, int max_wait_sec, unsigned int *skip_hid); MySrvC* find_server_in_hg(unsigned int _hid, const std::string& addr, int port); + +private: + void update_hostgroup_manager_mappings(); + uint64_t get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers = nullptr); + uint64_t get_mysql_servers_incoming_checksum(SQLite3_result* incoming_mysql_servers, bool use_precalculated_checksum = true); }; #endif /* __CLASS_MYSQL_HOSTGROUPS_MANAGER_H */ diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 7be5700503..b1dbf70199 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -19,17 +19,20 @@ * the queries issued for generating the checksum for each of the target modules, for simpler reasoning, they should * also represent the actual resultset being received when issuing them, since this resultset is used for computing the * 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules: - * - 'runtime_mysql_servers': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups', - * 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'. + * - 'runtime_mysql_servers': tables 'mysql_servers' * - 'runtime_mysql_users'. * - 'runtime_mysql_query_rules'. - * + * - 'mysql_servers_incoming': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups', + * 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'. * IMPORTANT: For further clarify this means that it's important that the actual resultset produced by the intercepted * query preserve the filtering and ordering expressed in this queries. */ /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_servers'. See top comment for details. */ -#define CLUSTER_QUERY_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" +#define CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'mysql_servers_incoming'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_SERVERS_INCOMING "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_replication_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup" @@ -190,6 +193,7 @@ class ProxySQL_Node_Entry { ProxySQL_Checksum_Value_2 mysql_servers; ProxySQL_Checksum_Value_2 mysql_users; ProxySQL_Checksum_Value_2 proxysql_servers; + ProxySQL_Checksum_Value_2 mysql_servers_incoming; } checksums_values; uint64_t global_checksum; }; @@ -274,12 +278,15 @@ class ProxySQL_Cluster_Nodes { bool Update_Node_Metrics(char * _h, uint16_t _p, MYSQL_RES *_r, unsigned long long _response_time); bool Update_Global_Checksum(char * _h, uint16_t _p, MYSQL_RES *_r); bool Update_Node_Checksums(char * _h, uint16_t _p, MYSQL_RES *_r); + void Reset_Global_Checksums(bool lock); void update_prometheus_nodes_metrics(); SQLite3_result * dump_table_proxysql_servers(); SQLite3_result * stats_proxysql_servers_checksums(); SQLite3_result * stats_proxysql_servers_metrics(); void get_peer_to_sync_mysql_query_rules(char **host, uint16_t *port, char** ip_address); - void get_peer_to_sync_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address); + void get_peer_to_sync_runtime_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address); + void get_peer_to_sync_mysql_servers_incoming(char** host, uint16_t* port, char** peer_mysql_servers_incoming_checksum, + char** peer_runtime_mysql_servers_checksum, char** ip_address); void get_peer_to_sync_mysql_users(char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_mysql_variables(char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_admin_variables(char **host, uint16_t* port, char** ip_address); @@ -366,6 +373,12 @@ struct variable_type { }; }; +enum class mysql_servers_sync_algorithm { + runtime_mysql_servers_and_mysql_servers_incoming = 1, // sync runtime_mysql_servers and mysql_server_incoming from remote peer + mysql_servers_incoming = 2, // sync mysql_server_incoming from remote peer + auto_select = 3 // based on -M flag +}; + /** * @brief Simple struct for holding a query, and three messages to report * the progress of the query execution. @@ -378,21 +391,23 @@ struct fetch_query { }; class ProxySQL_Cluster { - private: +private: + SQLite3DB* mydb; pthread_mutex_t mutex; std::vector term_threads; ProxySQL_Cluster_Nodes nodes; - char *cluster_username; - char *cluster_password; + char* cluster_username; + char* cluster_password; struct { - std::array p_counter_array {}; - std::array p_gauge_array {}; + std::array p_counter_array{}; + std::array p_gauge_array{}; } metrics; int fetch_and_store(MYSQL* conn, const fetch_query& f_query, MYSQL_RES** result); friend class ProxySQL_Node_Entry; - public: +public: pthread_mutex_t update_mysql_query_rules_mutex; - pthread_mutex_t update_mysql_servers_mutex; + pthread_mutex_t update_runtime_mysql_servers_mutex; + pthread_mutex_t update_mysql_servers_incoming_mutex; pthread_mutex_t update_mysql_users_mutex; pthread_mutex_t update_mysql_variables_mutex; pthread_mutex_t update_proxysql_servers_mutex; @@ -407,7 +422,7 @@ class ProxySQL_Cluster { */ SQLite3_result* proxysql_servers_to_monitor; - char *admin_mysql_ifaces; + char* admin_mysql_ifaces; int cluster_check_interval_ms; int cluster_check_status_frequency; int cluster_mysql_query_rules_diffs_before_sync; @@ -417,6 +432,7 @@ class ProxySQL_Cluster { int cluster_mysql_variables_diffs_before_sync; int cluster_ldap_variables_diffs_before_sync; int cluster_admin_variables_diffs_before_sync; + int cluster_mysql_servers_sync_algorithm; bool cluster_mysql_query_rules_save_to_disk; bool cluster_mysql_servers_save_to_disk; bool cluster_mysql_users_save_to_disk; @@ -428,7 +444,7 @@ class ProxySQL_Cluster { ~ProxySQL_Cluster(); void init() {}; void print_version(); - void load_servers_list(SQLite3_result *r, bool _lock = true) { + void load_servers_list(SQLite3_result* r, bool _lock = true) { nodes.load_servers_list(r, _lock); } void update_table_proxysql_servers_for_monitor(SQLite3_result* resultset) { @@ -440,33 +456,38 @@ class ProxySQL_Cluster { MySQL_Monitor::trigger_dns_cache_update(); } - void get_credentials(char **, char **); - void set_username(char *); - void set_password(char *); - void set_admin_mysql_ifaces(char *); - bool Update_Node_Metrics(char * _h, uint16_t _p, MYSQL_RES *_r, unsigned long long _response_time) { + void get_credentials(char**, char**); + void set_username(char*); + void set_password(char*); + void set_admin_mysql_ifaces(char*); + bool Update_Node_Metrics(char* _h, uint16_t _p, MYSQL_RES* _r, unsigned long long _response_time) { return nodes.Update_Node_Metrics(_h, _p, _r, _response_time); } - bool Update_Global_Checksum(char * _h, uint16_t _p, MYSQL_RES *_r) { + bool Update_Global_Checksum(char* _h, uint16_t _p, MYSQL_RES* _r) { return nodes.Update_Global_Checksum(_h, _p, _r); } - bool Update_Node_Checksums(char * _h, uint16_t _p, MYSQL_RES *_r = NULL) { + bool Update_Node_Checksums(char* _h, uint16_t _p, MYSQL_RES* _r = NULL) { return nodes.Update_Node_Checksums(_h, _p, _r); } - SQLite3_result *dump_table_proxysql_servers() { + void Reset_Global_Checksums(bool lock) { + nodes.Reset_Global_Checksums(lock); + } + SQLite3_result* dump_table_proxysql_servers() { return nodes.dump_table_proxysql_servers(); } - SQLite3_result * get_stats_proxysql_servers_checksums() { + SQLite3_result* get_stats_proxysql_servers_checksums() { return nodes.stats_proxysql_servers_checksums(); } - SQLite3_result * get_stats_proxysql_servers_metrics() { + SQLite3_result* get_stats_proxysql_servers_metrics() { return nodes.stats_proxysql_servers_metrics(); } void p_update_metrics(); void thread_ending(pthread_t); void join_term_thread(); void pull_mysql_query_rules_from_peer(const std::string& expected_checksum, const time_t epoch); - void pull_mysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch); + void pull_runtime_mysql_servers_from_peer(const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); + void pull_mysql_servers_incoming_from_peer(const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming, + const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, bool fetch_runtime_mysql_servers = false); void pull_mysql_users_from_peer(const std::string& expected_checksum, const time_t epoch); /** * @brief Pulls from peer the specified global variables by the type parameter. diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 6d1ddf5643..ca3a3e4e04 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -117,6 +117,7 @@ struct admin_metrics_map_idx { extern int admin__web_verbosity; struct incoming_servers_t { + SQLite3_result* incoming_mysql_servers = NULL; SQLite3_result* runtime_mysql_servers = NULL; SQLite3_result* incoming_replication_hostgroups = NULL; SQLite3_result* incoming_group_replication_hostgroups = NULL; @@ -125,9 +126,27 @@ struct incoming_servers_t { SQLite3_result* incoming_hostgroup_attributes = NULL; incoming_servers_t(); - incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); + incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); }; +// Seperate structs for runtime mysql server and mysql server incoming to avoid human error +struct runtime_mysql_servers_checksum_t { + std::string checksum; + time_t epoch; + + runtime_mysql_servers_checksum_t(); + runtime_mysql_servers_checksum_t(const std::string& checksum, time_t epoch); +}; + +struct mysql_servers_incoming_checksum_t { + std::string checksum; + time_t epoch; + + mysql_servers_incoming_checksum_t(); + mysql_servers_incoming_checksum_t(const std::string& checksum, time_t epoch); +}; +// + class ProxySQL_Admin { private: volatile int main_shutdown; @@ -185,6 +204,7 @@ class ProxySQL_Admin { int cluster_mysql_variables_diffs_before_sync; int cluster_admin_variables_diffs_before_sync; int cluster_ldap_variables_diffs_before_sync; + int cluster_mysql_servers_sync_algorithm; bool cluster_mysql_query_rules_save_to_disk; bool cluster_mysql_servers_save_to_disk; bool cluster_mysql_users_save_to_disk; @@ -275,8 +295,8 @@ class ProxySQL_Admin { void flush_mysql_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum = "", const time_t epoch = 0); char **get_variables_list(); - bool set_variable(char *name, char *value); - void flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum = "", const time_t epoch = 0); + bool set_variable(char *name, char *value, bool lock = true); + void flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum = "", const time_t epoch = 0, bool lock = true); void flush_admin_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false); void disk_upgrade_mysql_query_rules(); void disk_upgrade_mysql_servers(); @@ -398,7 +418,8 @@ class ProxySQL_Admin { // void flush_admin_variables__from_disk_to_memory(); // commented in 2.3 because unused void flush_admin_variables__from_memory_to_disk(); void flush_ldap_variables__from_memory_to_disk(); - void load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers = {}, const std::string& checksum = "", const time_t epoch = 0); + void load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers = {}, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, + const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming = {}); void save_mysql_servers_from_runtime(); /** * @brief Performs the load to runtime of the current configuration in 'main' for 'mysql_query_rules' and @@ -436,7 +457,8 @@ class ProxySQL_Admin { void load_scheduler_to_runtime(); void save_scheduler_runtime_to_database(bool); - void load_admin_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0) { flush_admin_variables___database_to_runtime(admindb, true, checksum, epoch); } + void load_admin_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0, bool lock = true) { + flush_admin_variables___database_to_runtime(admindb, true, checksum, epoch, lock); } void save_admin_variables_from_runtime() { flush_admin_variables___runtime_to_database(admindb, true, true, false); } void load_or_update_global_settings(SQLite3DB *); diff --git a/include/proxysql_glovars.hpp b/include/proxysql_glovars.hpp index 4d7d0529df..66c9424e0e 100644 --- a/include/proxysql_glovars.hpp +++ b/include/proxysql_glovars.hpp @@ -133,6 +133,7 @@ class ProxySQL_GlobalVariables { ProxySQL_Checksum_Value mysql_variables; ProxySQL_Checksum_Value ldap_variables; ProxySQL_Checksum_Value proxysql_servers; + ProxySQL_Checksum_Value mysql_servers_incoming; uint64_t global_checksum; unsigned long long updates_cnt; unsigned long long dumped_at; diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index afa9ee3a05..8448a607e0 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -140,6 +140,7 @@ enum debug_module { PROXY_DEBUG_QUERY_STATISTICS, PROXY_DEBUG_RESTAPI, PROXY_DEBUG_MONITOR, + PROXY_DEBUG_CLUSTER, PROXY_DEBUG_UNKNOWN // this module doesn't exist. It is used only to define the last possible module }; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 6f9c1a6546..8f7a28e535 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1356,6 +1356,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { incoming_galera_hostgroups=NULL; incoming_aws_aurora_hostgroups = NULL; incoming_hostgroup_attributes = NULL; + incoming_mysql_servers = NULL; pthread_rwlock_init(>id_rwlock, NULL); gtid_missing_nodes = false; gtid_ev_loop=NULL; @@ -1494,6 +1495,241 @@ unsigned int MySQL_HostGroups_Manager::get_servers_table_version() { return __sync_fetch_and_add(&status.servers_table_version,0); } +/** + * @brief Generates runtime mysql server records and checksum. + * + * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in + * the specified order. + * @param can be null or previously generated runtime_mysql_servers resultset can be passed. + */ +void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, + const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + // if any server has gtid_port enabled, use_gtid is set to true + // and then has_gtid_port is set too + bool use_gtid = false; + + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + const char* query = "SELECT mem_pointer, t1.hostgroup_id, t1.hostname, t1.port FROM mysql_servers t1 LEFT OUTER JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE t2.hostgroup_id IS NULL"; + mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } + else { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Dumping mysql_servers LEFT JOIN mysql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + long long ptr = atoll(r->fields[0]); + proxy_warning("Removed server at address %lld, hostgroup %s, address %s port %s. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", ptr, r->fields[1], r->fields[2], r->fields[3]); + MySrvC* mysrvc = (MySrvC*)ptr; + mysrvc->status = MYSQL_SERVER_STATUS_OFFLINE_HARD; + mysrvc->ConnectionsFree->drop_all_connections(); + char* q1 = (char*)"DELETE FROM mysql_servers WHERE mem_pointer=%lld"; + char* q2 = (char*)malloc(strlen(q1) + 32); + sprintf(q2, q1, ptr); + mydb->execute(q2); + free(q2); + } + } + if (resultset) { delete resultset; resultset = NULL; } + + mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); + + // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) + query = (char*)"SELECT t1.*, t2.gtid_port, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.gtid_port<>t2.gtid_port OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } + else { + + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Dumping mysql_servers JOIN mysql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + // optimization #829 + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement2 = NULL; + //sqlite3 *mydb3=mydb->get_db(); + char* query1 = (char*)"UPDATE mysql_servers SET mem_pointer = ?1 WHERE hostgroup_id = ?2 AND hostname = ?3 AND port = ?4"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = mydb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, mydb); + char* query2 = (char*)"UPDATE mysql_servers SET weight = ?1 , status = ?2 , compression = ?3 , max_connections = ?4 , max_replication_lag = ?5 , use_ssl = ?6 , max_latency_ms = ?7 , comment = ?8 , gtid_port = ?9 WHERE hostgroup_id = ?10 AND hostname = ?11 AND port = ?12"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query2, -1, &statement2, 0); + rc = mydb->prepare_v2(query2, &statement2); + ASSERT_SQLITE_OK(rc, mydb); + + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + long long ptr = atoll(r->fields[12]); // increase this index every time a new column is added + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), ptr, atoi(r->fields[0]), atoi(r->fields[6])); + //fprintf(stderr,"%lld\n", ptr); + if (ptr == 0) { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5])); + } + MySrvC* mysrvc = new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), atoi(r->fields[10]), r->fields[11]); // add new fields here if adding more columns in mysql_servers + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), mysrvc, atoi(r->fields[0])); + MyHGM->add(mysrvc, atoi(r->fields[0])); + ptr = (uintptr_t)mysrvc; + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, ptr); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_text)(statement1, 3, r->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); + if (mysrvc->gtid_port) { + // this server has gtid_port configured, we set use_gtid + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); + use_gtid = true; + } + } + else { + bool run_update = false; + MySrvC* mysrvc = (MySrvC*)ptr; + // carefully increase the 2nd index by 1 for every new column added + if (atoi(r->fields[3]) != atoi(r->fields[13])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing gtid_port for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), mysrvc->gtid_port, atoi(r->fields[13])); + mysrvc->gtid_port = atoi(r->fields[13]); + } + + if (atoi(r->fields[4]) != atoi(r->fields[14])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), mysrvc->weight, atoi(r->fields[14])); + mysrvc->weight = atoi(r->fields[14]); + } + if (atoi(r->fields[5]) != atoi(r->fields[15])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing status for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[5]), mysrvc->status, atoi(r->fields[15])); + mysrvc->status = (MySerStatus)atoi(r->fields[15]); + if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED) { + mysrvc->shunned_automatic = false; + } + } + if (atoi(r->fields[6]) != atoi(r->fields[16])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing compression for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[6]), mysrvc->compression, atoi(r->fields[16])); + mysrvc->compression = atoi(r->fields[16]); + } + if (atoi(r->fields[7]) != atoi(r->fields[17])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_connections for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[7]), mysrvc->max_connections, atoi(r->fields[17])); + mysrvc->max_connections = atoi(r->fields[17]); + } + if (atoi(r->fields[8]) != atoi(r->fields[18])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_replication_lag for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[8]), mysrvc->max_replication_lag, atoi(r->fields[18])); + mysrvc->max_replication_lag = atoi(r->fields[18]); + if (mysrvc->max_replication_lag == 0) { // we just changed it to 0 + if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + // the server is currently shunned due to replication lag + // but we reset max_replication_lag to 0 + // therefore we immediately reset the status too + mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; + } + } + } + if (atoi(r->fields[9]) != atoi(r->fields[19])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing use_ssl for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[9]), mysrvc->use_ssl, atoi(r->fields[19])); + mysrvc->use_ssl = atoi(r->fields[19]); + } + if (atoi(r->fields[10]) != atoi(r->fields[20])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_latency_ms for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[10]), mysrvc->max_latency_us / 1000, atoi(r->fields[20])); + mysrvc->max_latency_us = 1000 * atoi(r->fields[20]); + } + if (strcmp(r->fields[11], r->fields[21])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing comment for server %d:%s:%d (%s:%d) from '%s' to '%s'\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[11], r->fields[21]); + free(mysrvc->comment); + mysrvc->comment = strdup(r->fields[21]); + } + if (run_update) { + rc = (*proxy_sqlite3_bind_int64)(statement2, 1, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 2, mysrvc->status); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 3, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 4, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 5, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 6, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 7, mysrvc->max_latency_us / 1000); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_text)(statement2, 8, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 9, mysrvc->gtid_port); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 10, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_text)(statement2, 11, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement2, 12, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement2); + rc = (*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, mydb); + } + if (mysrvc->gtid_port) { + // this server has gtid_port configured, we set use_gtid + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); + use_gtid = true; + } + } + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement2); + } + if (use_gtid) { + has_gtid_port = true; + } + else { + has_gtid_port = false; + } + if (resultset) { delete resultset; resultset = NULL; } + + purge_mysql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + const auto mysql_servers_checksum = get_mysql_servers_checksum(); + + update_hostgroup_manager_mappings(); + + char buf[80]; + uint32_t d32[2]; + memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); + pthread_mutex_lock(&GloVars.checksum_mutex); + GloVars.checksums_values.mysql_servers.set_checksum(buf); + GloVars.checksums_values.mysql_servers.version++; + //struct timespec ts; + //clock_gettime(CLOCK_REALTIME, &ts); + time_t t = time(NULL); + + if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && + GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { + GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; + } + else { + GloVars.checksums_values.mysql_servers.epoch = t; + } + + GloVars.checksums_values.updates_cnt++; + GloVars.generate_global_checksum(); + GloVars.epoch_version = t; + pthread_mutex_unlock(&GloVars.checksum_mutex); + + update_table_mysql_servers_for_monitor(false); +} + // we always assume that the calling thread has acquired a rdlock() int MySQL_HostGroups_Manager::servers_add(SQLite3_result *resultset) { if (resultset==NULL) { @@ -1586,7 +1822,7 @@ SQLite3_result * MySQL_HostGroups_Manager::execute_query(char *query, char **err return resultset; } -void MySQL_HostGroups_Manager::CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum) { +void MySQL_HostGroups_Manager::CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum) { char *error=NULL; int cols=0; int affected_rows=0; @@ -1595,14 +1831,8 @@ void MySQL_HostGroups_Manager::CUCFT1(SpookyHash& myhash, bool& init, const stri mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); if (resultset) { if (resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = resultset->raw_checksum(); - raw_checksum = hash1_; - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), hash1_); + raw_checksum = resultset->raw_checksum(); + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), raw_checksum); } delete resultset; } else { @@ -1610,16 +1840,82 @@ void MySQL_HostGroups_Manager::CUCFT1(SpookyHash& myhash, bool& init, const stri } } -void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& myhash, bool& init) { - CUCFT1(myhash,init,"mysql_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); - CUCFT1(myhash,init,"mysql_group_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GROUP_REPLICATION_HOSTGROUPS]); - CUCFT1(myhash,init,"mysql_galera_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GALERA_HOSTGROUPS]); - CUCFT1(myhash,init,"mysql_aws_aurora_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_AURORA_HOSTGROUPS]); - CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); +void MySQL_HostGroups_Manager::commit_update_checksums_from_tables() { + CUCFT1("mysql_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); + CUCFT1("mysql_group_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GROUP_REPLICATION_HOSTGROUPS]); + CUCFT1("mysql_galera_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GALERA_HOSTGROUPS]); + CUCFT1("mysql_aws_aurora_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_AURORA_HOSTGROUPS]); + CUCFT1("mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); +} + +void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { + + if (hgsm_mysql_servers_checksum != table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] || + hgsm_mysql_replication_hostgroups_checksum != table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]) + { + proxy_info("Rebuilding 'Hostgroup_Manager_Mapping' due to checksums change - mysql_servers { old: 0x%lX, new: 0x%lX }, mysql_replication_hostgroups { old:0x%lX, new:0x%lX }\n", + hgsm_mysql_servers_checksum, table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS], + hgsm_mysql_replication_hostgroups_checksum, table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + hostgroup_server_mapping.clear(); + + const char* query = "SELECT DISTINCT hostname, port, '1' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM mysql_replication_hostgroups JOIN mysql_servers ON hostgroup_id=writer_hostgroup WHERE status<>3 \ + UNION \ + SELECT DISTINCT hostname, port, '0' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM mysql_replication_hostgroups JOIN mysql_servers ON hostgroup_id=reader_hostgroup WHERE status<>3 \ + ORDER BY hostname, port"; + + mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + + if (resultset && resultset->rows_count) { + std::string fetched_server_id; + HostGroup_Server_Mapping* fetched_server_mapping = NULL; + + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + + const std::string& server_id = std::string(r->fields[0]) + ":::" + r->fields[1]; + + if (fetched_server_mapping == NULL || server_id != fetched_server_id) { + + auto itr = hostgroup_server_mapping.find(server_id); + + if (itr == hostgroup_server_mapping.end()) { + std::unique_ptr server_mapping(new HostGroup_Server_Mapping(this)); + fetched_server_mapping = server_mapping.get(); + hostgroup_server_mapping.insert({ server_id, std::move(server_mapping) }); + } + else { + fetched_server_mapping = itr->second.get(); + } + + fetched_server_id = server_id; + } + + HostGroup_Server_Mapping::Node node; + node.server_status = static_cast(atoi(r->fields[3])); + node.reader_hostgroup_id = atoi(r->fields[4]); + node.writer_hostgroup_id = atoi(r->fields[5]); + node.srv = reinterpret_cast(atoll(r->fields[6])); + + HostGroup_Server_Mapping::Type type = (r->fields[2] && r->fields[2][0] == '1') ? HostGroup_Server_Mapping::Type::WRITER : HostGroup_Server_Mapping::Type::READER; + fetched_server_mapping->add(type, node); + } + } + delete resultset; + + hgsm_mysql_servers_checksum = table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; + hgsm_mysql_replication_hostgroups_checksum = table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]; + } } bool MySQL_HostGroups_Manager::commit( - SQLite3_result* runtime_mysql_servers, const std::string& checksum, const time_t epoch + SQLite3_result* runtime_mysql_servers, SQLite3_result* mysql_servers_incoming, + const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming ) { unsigned long long curtime1=monotonic_time(); @@ -1830,8 +2126,8 @@ bool MySQL_HostGroups_Manager::commit( has_gtid_port = false; } if (resultset) { delete resultset; resultset=NULL; } - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); - mydb->execute("DELETE FROM mysql_servers_incoming"); + //proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); + //mydb->execute("DELETE FROM mysql_servers_incoming"); // replication if (incoming_replication_hostgroups) { // this IF is extremely important, otherwise replication hostgroups may disappear @@ -1871,91 +2167,47 @@ bool MySQL_HostGroups_Manager::commit( //if (GloAdmin && GloAdmin->checksum_variables.checksum_mysql_servers) { - uint64_t hash1 = 0, hash2 = 0; - SpookyHash myhash; - char buf[80]; - bool init = false; - { - mydb->execute("DELETE FROM mysql_servers"); - generate_mysql_servers_table(); - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (runtime_mysql_servers == nullptr) { - char* error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - - mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); - - // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster - // nodes, or relevant for checksum computation. - const size_t init_row_count = resultset->rows_count; - size_t rm_rows_count = 0; - const auto is_offline_server = [&rm_rows_count] (SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - rm_rows_count += 1; - return true; - } else { - return false; - } - }; - resultset->rows.erase( - std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), - resultset->rows.end() - ); - resultset->rows_count = init_row_count - rm_rows_count; - - save_runtime_mysql_servers(resultset); - } else { - save_runtime_mysql_servers(runtime_mysql_servers); - } - - // reset all checksum - table_resultset_checksum.fill(0); - - if (resultset) { - if (resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = resultset->raw_checksum(); - - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); - } - delete resultset; - } else { - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); - } - } + const auto mysql_servers_checksum = get_mysql_servers_checksum(runtime_mysql_servers); + const auto mysql_servers_incoming_checksum = get_mysql_servers_incoming_checksum(mysql_servers_incoming, false); - commit_update_checksums_from_tables(myhash, init); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); + mydb->execute("DELETE FROM mysql_servers_incoming"); - if (init == true) { - myhash.Final(&hash1, &hash2); - } + char buf[80]; uint32_t d32[2]; - memcpy(&d32,&hash1,sizeof(hash1)); - sprintf(buf,"0x%0X%0X", d32[0], d32[1]); + const time_t t = time(NULL); + + memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); pthread_mutex_lock(&GloVars.checksum_mutex); GloVars.checksums_values.mysql_servers.set_checksum(buf); GloVars.checksums_values.mysql_servers.version++; //struct timespec ts; //clock_gettime(CLOCK_REALTIME, &ts); - time_t t = time(NULL); - if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_servers.checksum == checksum) { - GloVars.checksums_values.mysql_servers.epoch = epoch; - } else { + if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && + GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { + GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; + } + else { GloVars.checksums_values.mysql_servers.epoch = t; } + + memcpy(&d32, &mysql_servers_incoming_checksum, sizeof(mysql_servers_incoming_checksum)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); + GloVars.checksums_values.mysql_servers_incoming.set_checksum(buf); + GloVars.checksums_values.mysql_servers_incoming.version++; + + if (peer_mysql_server_incoming.epoch != 0 && peer_mysql_server_incoming.checksum.empty() == false && + GloVars.checksums_values.mysql_servers_incoming.checksum == peer_mysql_server_incoming.checksum) { + GloVars.checksums_values.mysql_servers_incoming.epoch = peer_mysql_server_incoming.epoch; + } + else { + GloVars.checksums_values.mysql_servers_incoming.epoch = t; + } + GloVars.checksums_values.updates_cnt++; GloVars.generate_global_checksum(); GloVars.epoch_version = t; @@ -1963,67 +2215,7 @@ bool MySQL_HostGroups_Manager::commit( } // fill Hostgroup_Manager_Mapping with latest records - if (hgsm_mysql_servers_checksum != table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] || - hgsm_mysql_replication_hostgroups_checksum != table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]) - { - proxy_info("Rebuilding 'Hostgroup_Manager_Mapping' due to checksums change - mysql_servers { old: 0x%lX, new: 0x%lX }, mysql_replication_hostgroups { old:0x%lX, new:0x%lX }\n", - hgsm_mysql_servers_checksum, table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS], - hgsm_mysql_replication_hostgroups_checksum, table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); - - char* error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - - const char* query = "SELECT DISTINCT hostname, port, '1' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM mysql_replication_hostgroups JOIN mysql_servers ON hostgroup_id=writer_hostgroup WHERE status<>3 \ - UNION \ - SELECT DISTINCT hostname, port, '0' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM mysql_replication_hostgroups JOIN mysql_servers ON hostgroup_id=reader_hostgroup WHERE status<>3 \ - ORDER BY hostname, port"; - - mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); - - hostgroup_server_mapping.clear(); - - if (resultset && resultset->rows_count) { - std::string fetched_server_id; - HostGroup_Server_Mapping* fetched_server_mapping = NULL; - - for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { - SQLite3_row* r = *it; - - const std::string& server_id = std::string(r->fields[0]) + ":::" + r->fields[1]; - - if (fetched_server_mapping == NULL || server_id != fetched_server_id) { - - auto itr = hostgroup_server_mapping.find(server_id); - - if (itr == hostgroup_server_mapping.end()) { - std::unique_ptr server_mapping(new HostGroup_Server_Mapping(this)); - fetched_server_mapping = server_mapping.get(); - hostgroup_server_mapping.insert({ server_id, std::move(server_mapping) }); - } - else { - fetched_server_mapping = itr->second.get(); - } - - fetched_server_id = server_id; - } - - HostGroup_Server_Mapping::Node node; - node.server_status = static_cast(atoi(r->fields[3])); - node.reader_hostgroup_id = atoi(r->fields[4]); - node.writer_hostgroup_id = atoi(r->fields[5]); - node.srv = reinterpret_cast(atoll(r->fields[6])); - - HostGroup_Server_Mapping::Type type = (r->fields[2] && r->fields[2][0] == '1') ? HostGroup_Server_Mapping::Type::WRITER : HostGroup_Server_Mapping::Type::READER; - fetched_server_mapping->add(type, node); - } - } - delete resultset; - - hgsm_mysql_servers_checksum = table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; - hgsm_mysql_replication_hostgroups_checksum = table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]; - } + update_hostgroup_manager_mappings(); ev_async_send(gtid_ev_loop, gtid_ev_async); @@ -2057,6 +2249,261 @@ bool MySQL_HostGroups_Manager::commit( return true; } +uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers) { + + //Note: GloVars.checksum_mutex needs to be locked + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + char* query = (char*)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; + mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + + if (runtime_mysql_servers == nullptr) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster + // nodes, or relevant for checksum computation. + const size_t init_row_count = resultset->rows_count; + size_t rm_rows_count = 0; + const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + rm_rows_count += 1; + return true; + } + else { + return false; + } + }; + resultset->rows.erase( + std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), + resultset->rows.end() + ); + resultset->rows_count = init_row_count - rm_rows_count; + + save_runtime_mysql_servers(resultset); + } + else { + save_runtime_mysql_servers(runtime_mysql_servers); + } + + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = 0; + + if (resultset) { + if (resultset->rows_count) { + uint64_t hash1_ = resultset->raw_checksum(); + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); + } + delete resultset; + } else { + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); + } + + return table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; +} + +uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_result* incoming_mysql_servers, + bool use_precalculated_checksum) { + + //Note: GloVars.checksum_mutex needs to be locked + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + if (incoming_mysql_servers == nullptr) + { + mydb->execute_statement(MYHGM_GEN_INCOMING_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); + if (resultset) { + if (resultset->rows_count) { + + const size_t init_row_count = resultset->rows_count; + size_t rm_rows_count = 0; + const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + rm_rows_count += 1; + return true; + } + else { + return false; + } + }; + resultset->rows.erase( + std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), + resultset->rows.end() + ); + resultset->rows_count = init_row_count - rm_rows_count; + + save_mysql_servers_incoming(resultset); + } + } + else { + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", (long unsigned int)0); + } + } else { + save_mysql_servers_incoming(incoming_mysql_servers); + } + + if (use_precalculated_checksum == false) { + // reset all checksum + table_resultset_checksum.fill(0); + } + + resultset = get_current_mysql_table("mysql_servers_incoming"); + + if (resultset) { + int status_idx = -1; + + for (int i = 0; i < resultset->columns; i++) { + if (resultset->column_definition[i] && resultset->column_definition[i]->name && + strcasecmp(resultset->column_definition[i]->name, "status") == 0) { + status_idx = i; + break; + } + } + + if (resultset->rows_count) { + uint64_t hash1 = 0, hash2 = 0; + SpookyHash myhash; + myhash.Init(19, 3); + + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + + const char* status_conv = nullptr; + const char* status = r->fields[status_idx]; + + if (status) { + if (strcasecmp(status, "OFFLINE_HARD") == 0) + continue; + + if (strcasecmp(status, "ONLINE") == 0 || + strcasecmp(status, "SHUNNED") == 0) { + status_conv = "0"; + } + else if (strcasecmp(status, "OFFLINE_SOFT") == 0) { + status_conv = "2"; + } + } + + for (int i = 0; i < resultset->columns; i++) { + + if (i != status_idx) { + if (r->fields[i]) { + myhash.Update(r->fields[i], r->sizes[i]); + } else { + myhash.Update("", 0); + } + } else { + if (status_conv) { + myhash.Update(status_conv, strlen(status_conv)); + } else { + myhash.Update("", 0); + } + } + } + } + myhash.Final(&hash1, &hash2); + + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_INCOMING] = hash1; + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", hash1); + } + } + + + /*char* query = (char*)"SELECT hostgroup_id,hostname,port,gtid_port,CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, \ + weight,compression,max_connections, max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_incoming WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; + mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + if (resultset) { + if (resultset->rows_count) { + uint64_t hash1_ = resultset->raw_checksum(); + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_INCOMING] = hash1_; + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", hash1_); + } + }*/ + + if (use_precalculated_checksum == false) { + commit_update_checksums_from_tables(); + } + + uint64_t hash1 = 0, hash2 = 0; + SpookyHash myhash; + bool init = false; + + hash1 = table_resultset_checksum[MYSQL_SERVERS_INCOMING]; + if (hash1) { + if (init == false) { + init = true; + myhash.Init(19, 3); + } + + myhash.Update(&hash1, sizeof(hash1)); + } + + hash1 = table_resultset_checksum[MYSQL_REPLICATION_HOSTGROUPS]; + if (hash1) { + if (init == false) { + init = true; + myhash.Init(19, 3); + } + + myhash.Update(&hash1, sizeof(hash1)); + } + + hash1 = table_resultset_checksum[MYSQL_GROUP_REPLICATION_HOSTGROUPS]; + if (hash1) { + if (init == false) { + init = true; + myhash.Init(19, 3); + } + + myhash.Update(&hash1, sizeof(hash1)); + } + + hash1 = table_resultset_checksum[MYSQL_GALERA_HOSTGROUPS]; + if (hash1) { + if (init == false) { + init = true; + myhash.Init(19, 3); + } + + myhash.Update(&hash1, sizeof(hash1)); + } + + hash1 = table_resultset_checksum[MYSQL_AWS_AURORA_HOSTGROUPS]; + if (hash1) { + if (init == false) { + init = true; + myhash.Init(19, 3); + } + + myhash.Update(&hash1, sizeof(hash1)); + } + + hash1 = table_resultset_checksum[MYSQL_HOSTGROUP_ATTRIBUTES]; + if (hash1) { + if (init == false) { + init = true; + myhash.Init(19, 3); + } + + myhash.Update(&hash1, sizeof(hash1)); + } + + if (init == true) { + myhash.Final(&hash1, &hash2); + } + + return hash1; +} + bool MySQL_HostGroups_Manager::gtid_exists(MySrvC *mysrvc, char * gtid_uuid, uint64_t gtid_trxid) { bool ret = false; pthread_rwlock_rdlock(>id_rwlock); @@ -3830,6 +4277,14 @@ void MySQL_HostGroups_Manager::save_runtime_mysql_servers(SQLite3_result *s) { runtime_mysql_servers=s; } +void MySQL_HostGroups_Manager::save_mysql_servers_incoming(SQLite3_result* s) { + if (incoming_mysql_servers) { + delete incoming_mysql_servers; + incoming_mysql_servers = nullptr; + } + incoming_mysql_servers = s; +} + SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& name) { if (name == "mysql_aws_aurora_hostgroups") { return this->incoming_aws_aurora_hostgroups; @@ -3843,6 +4298,8 @@ SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& return this->incoming_hostgroup_attributes; } else if (name == "mysql_servers") { return this->runtime_mysql_servers; + } else if (name == "mysql_servers_incoming") { + return this->incoming_mysql_servers; } else { assert(0); } @@ -4710,72 +5167,30 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listexecute("DELETE FROM mysql_servers"); generate_mysql_servers_table(); - //if (GloAdmin && GloAdmin->checksum_variables.checksum_mysql_servers) - { - char* error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); - save_runtime_mysql_servers(resultset); // assigning runtime_mysql_servers with updated mysql server resultset - - resultset = NULL; - - // reset mysql_server checksum - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = 0; - hgsm_mysql_servers_checksum = 0; - - char* query = (char*)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; - mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + // reset hgsm_mysql_servers_checksum checksum + hgsm_mysql_servers_checksum = 0; - if (resultset) { - if (resultset->rows_count) { - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = resultset->raw_checksum(); - - hgsm_mysql_servers_checksum = table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; - } - delete resultset; - } else { - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); - } + const auto mysql_servers_checksum = get_mysql_servers_checksum(); - uint64_t hash = 0, hash2 = 0; - SpookyHash myhash; - bool init = false; - - for (uint64_t hash_val : table_resultset_checksum) { - if (hash_val) { - if (init == false) { - init = true; - myhash.Init(19, 3); - } - - myhash.Update(&hash_val, sizeof(hash_val)); - } - } - - if (init == true) { - myhash.Final(&hash, &hash2); - } + hgsm_mysql_servers_checksum = mysql_servers_checksum; - char buf[80]; - uint32_t d32[2]; - memcpy(&d32, &hash, sizeof(hash)); - sprintf(buf, "0x%0X%0X", d32[0], d32[1]); - pthread_mutex_lock(&GloVars.checksum_mutex); - GloVars.checksums_values.mysql_servers.set_checksum(buf); - GloVars.checksums_values.mysql_servers.version++; - //struct timespec ts; - //clock_gettime(CLOCK_REALTIME, &ts); - time_t t = time(NULL); + char buf[80]; + uint32_t d32[2]; + memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); + pthread_mutex_lock(&GloVars.checksum_mutex); + GloVars.checksums_values.mysql_servers.set_checksum(buf); + GloVars.checksums_values.mysql_servers.version++; + //struct timespec ts; + //clock_gettime(CLOCK_REALTIME, &ts); + time_t t = time(NULL); - GloVars.checksums_values.mysql_servers.epoch = t; + GloVars.checksums_values.mysql_servers.epoch = t; - GloVars.checksums_values.updates_cnt++; - GloVars.generate_global_checksum(); - GloVars.epoch_version = t; - pthread_mutex_unlock(&GloVars.checksum_mutex); - } + GloVars.checksums_values.updates_cnt++; + GloVars.generate_global_checksum(); + GloVars.epoch_version = t; + pthread_mutex_unlock(&GloVars.checksum_mutex); } wrunlock(); unsigned long long curtime2 = monotonic_time(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index a34536f139..0e53edd36c 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -658,6 +658,7 @@ static char * admin_variables_names[]= { (char *)"cluster_mysql_variables_save_to_disk", (char *)"cluster_admin_variables_save_to_disk", (char *)"cluster_ldap_variables_save_to_disk", + (char *)"cluster_mysql_servers_sync_algorithm", (char *)"checksum_mysql_query_rules", (char *)"checksum_mysql_servers", (char *)"checksum_mysql_users", @@ -1057,21 +1058,34 @@ bool FlushCommandWrapper(MySQL_Session *sess, const string& modname, char *query incoming_servers_t::incoming_servers_t() {} incoming_servers_t::incoming_servers_t( - SQLite3_result* runtime_mysql_servers, + SQLite3_result* incoming_mysql_servers, SQLite3_result* incoming_replication_hostgroups, SQLite3_result* incoming_group_replication_hostgroups, SQLite3_result* incoming_galera_hostgroups, SQLite3_result* incoming_aurora_hostgroups, - SQLite3_result* incoming_hostgroup_attributes + SQLite3_result* incoming_hostgroup_attributes, + SQLite3_result* runtime_mysql_servers ) : - runtime_mysql_servers(runtime_mysql_servers), + incoming_mysql_servers(incoming_mysql_servers), incoming_replication_hostgroups(incoming_replication_hostgroups), incoming_group_replication_hostgroups(incoming_group_replication_hostgroups), incoming_galera_hostgroups(incoming_galera_hostgroups), incoming_aurora_hostgroups(incoming_aurora_hostgroups), - incoming_hostgroup_attributes(incoming_hostgroup_attributes) + incoming_hostgroup_attributes(incoming_hostgroup_attributes), + runtime_mysql_servers(runtime_mysql_servers) {} +runtime_mysql_servers_checksum_t::runtime_mysql_servers_checksum_t() : epoch(0) {} + +runtime_mysql_servers_checksum_t::runtime_mysql_servers_checksum_t(const std::string& checksum, time_t epoch) : + checksum(checksum), epoch(epoch) {} + +mysql_servers_incoming_checksum_t::mysql_servers_incoming_checksum_t() : epoch(0) {} + +mysql_servers_incoming_checksum_t::mysql_servers_incoming_checksum_t(const std::string& checksum, time_t epoch) : + checksum(checksum), epoch(epoch) {} + + int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { int r = 0; if (!GloQPro) return 0; @@ -3670,7 +3684,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats string tn = ""; - if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS))) { + if (!strncasecmp(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS))) { tn = "mysql_servers"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) { tn = "mysql_replication_hostgroups"; @@ -3682,6 +3696,8 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { tn = "mysql_aws_aurora_hostgroups"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES, query_no_space, strlen(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES))) { tn = "mysql_hostgroup_attributes"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_INCOMING, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_INCOMING))) { + tn = "mysql_servers_incoming"; } if (tn != "") { GloAdmin->mysql_servers_wrlock(); @@ -5763,6 +5779,7 @@ ProxySQL_Admin::ProxySQL_Admin() : variables.cluster_mysql_variables_diffs_before_sync = 3; variables.cluster_admin_variables_diffs_before_sync = 3; variables.cluster_ldap_variables_diffs_before_sync = 3; + variables.cluster_mysql_servers_sync_algorithm = 1; checksum_variables.checksum_mysql_query_rules = true; checksum_variables.checksum_mysql_servers = true; checksum_variables.checksum_mysql_users = true; @@ -6485,7 +6502,8 @@ void ProxySQL_Admin::load_or_update_global_settings(SQLite3DB *db) { } } -void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace, const string& checksum, const time_t epoch) { +void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace, const string& checksum, const time_t epoch, + bool lock) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ADMIN variables. Replace:%d\n", replace); char *error=NULL; int cols=0; @@ -6500,7 +6518,7 @@ void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, wrlock(); for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; - bool rc=set_variable(r->fields[0],r->fields[1]); + bool rc=set_variable(r->fields[0],r->fields[1],lock); if (rc==false) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Impossible to set variable %s with value \"%s\"\n", r->fields[0],r->fields[1]); if (replace) { @@ -7747,6 +7765,10 @@ char * ProxySQL_Admin::get_variable(char *name) { sprintf(intbuf,"%d",variables.cluster_ldap_variables_diffs_before_sync); return strdup(intbuf); } + if (!strcasecmp(name,"cluster_mysql_servers_sync_algorithm")) { + sprintf(intbuf, "%d", variables.cluster_mysql_servers_sync_algorithm); + return strdup(intbuf); + } if (!strcasecmp(name,"cluster_mysql_query_rules_save_to_disk")) { return strdup((variables.cluster_mysql_query_rules_save_to_disk ? "true" : "false")); } @@ -7876,7 +7898,7 @@ void ProxySQL_Admin::delete_credentials(char *credentials) { free_tokenizer( &tok ); } -bool ProxySQL_Admin::set_variable(char *name, char *value) { // this is the public function, accessible from admin +bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this is the public function, accessible from admin size_t vallen=strlen(value); if (!strcasecmp(name,"admin_credentials")) { @@ -8164,6 +8186,20 @@ bool ProxySQL_Admin::set_variable(char *name, char *value) { // this is the pub return false; } } + if (!strcasecmp(name,"cluster_mysql_servers_sync_algorithm")) { + int intv=atoi(value); + if (intv >= 1 && intv <= 3) { + variables.cluster_mysql_servers_sync_algorithm =intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_sync_algorithm, intv); + + proxy_info("'cluster_mysql_servers_sync_algorithm' updated. Resetting global checksums to force Cluster re-sync."); + GloProxyCluster->Reset_Global_Checksums(lock); + + return true; + } else { + return false; + } + } if (!strcasecmp(name,"version")) { if (strcasecmp(value,(char *)PROXYSQL_VERSION)==0) { return true; @@ -11021,6 +11057,14 @@ void ProxySQL_Admin::dump_checksums_values_table() { rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 1, "mysql_servers_incoming", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.mysql_servers_incoming.version); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.mysql_servers_incoming.epoch); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.mysql_servers_incoming.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + if (GloMyLdapAuth) { rc=(*proxy_sqlite3_bind_text)(statement1, 1, "ldap_variables", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.ldap_variables.version); ASSERT_SQLITE_OK(rc, admindb); @@ -11821,9 +11865,8 @@ void ProxySQL_Admin::load_scheduler_to_runtime() { resultset=NULL; } -void ProxySQL_Admin::load_mysql_servers_to_runtime( - const incoming_servers_t& incoming_servers, const std::string& checksum, const time_t epoch -) { +void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers, + const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming) { // make sure that the caller has called mysql_servers_wrlock() char *error=NULL; int cols=0; @@ -11835,6 +11878,8 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; SQLite3_result *resultset_hostgroup_attributes=NULL; + SQLite3_result *resultset_mysql_servers_incoming = NULL; + SQLite3_result* resultset_mysql_servers_admin = NULL; SQLite3_result* runtime_mysql_servers = incoming_servers.runtime_mysql_servers; SQLite3_result* incoming_replication_hostgroups = incoming_servers.incoming_replication_hostgroups; @@ -11842,11 +11887,15 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( SQLite3_result* incoming_galera_hostgroups = incoming_servers.incoming_galera_hostgroups; SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups; SQLite3_result* incoming_hostgroup_attributes = incoming_servers.incoming_hostgroup_attributes; + SQLite3_result* incoming_mysql_servers = incoming_servers.incoming_mysql_servers; char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset_mysql_servers_admin); + if (runtime_mysql_servers == nullptr) { - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); + resultset_servers = resultset_mysql_servers_admin; + resultset_mysql_servers_admin = nullptr; } else { resultset_servers = runtime_mysql_servers; } @@ -12010,7 +12059,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( } // commit all the changes - MyHGM->commit(runtime_mysql_servers, checksum, epoch); + MyHGM->commit(runtime_mysql_servers, incoming_mysql_servers, peer_runtime_mysql_server, peer_mysql_server_incoming); // quering runtime table will update and return latest records, so this is not needed. // GloAdmin->save_mysql_servers_runtime_to_database(true); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index c70c3b8959..c73f388215 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -66,6 +66,8 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { mysql_thread_init(); pthread_detach(pthread_self()); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Thread started for peer %s:%d\n", node->hostname, node->port); + proxy_info("Cluster: starting thread for peer %s:%d\n", node->hostname, node->port); char *query1 = (char *)"SELECT GLOBAL_CHECKSUM()"; // in future this will be used for "light check" char *query2 = (char *)"SELECT * FROM stats_mysql_global ORDER BY Variable_Name"; @@ -104,6 +106,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } //rc_conn = mysql_real_connect(conn, node->hostname, username, password, NULL, node->port, NULL, CLIENT_COMPRESS); // FIXME: add optional support for compression + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Connecting to peer %s:%d\n", node->hostname, node->port); rc_conn = mysql_real_connect(conn, node->get_host_address(), username, password, NULL, node->port, NULL, 0); // if (rc_conn) { // } @@ -122,6 +125,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { if (row[0]) { const char* PROXYSQL_VERSION_ = GloMyLdapAuth == nullptr ? PROXYSQL_VERSION : PROXYSQL_VERSION"-Enterprise"; if (strcmp(row[0], PROXYSQL_VERSION_)==0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Clustering with peer %s:%d . Remote version: %s . Self version: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION_); proxy_info("Cluster: clustering with peer %s:%d . Remote version: %s . Self version: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION_); same_version = true; std::string q = "PROXYSQL CLUSTER_NODE_UUID "; @@ -130,6 +134,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { pthread_mutex_lock(&GloProxyCluster->admin_mysql_ifaces_mutex); q += GloProxyCluster->admin_mysql_ifaces; pthread_mutex_unlock(&GloProxyCluster->admin_mysql_ifaces_mutex); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Sending CLUSTER_NODE_UUID %s to peer %s:%d\n", GloVars.uuid, node->hostname, node->port); proxy_info("Cluster: sending CLUSTER_NODE_UUID %s to peer %s:%d\n", GloVars.uuid, node->hostname, node->port); rc_query = mysql_query(conn, q.c_str()); } else { @@ -139,6 +144,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } mysql_free_result(result); if (same_version == false) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Remote peer %s:%d proxysql version is different. Closing connection\n", node->hostname, node->port); mysql_close(conn); conn = mysql_init(NULL); int exit_after_N_seconds = 30; // hardcoded sleep time @@ -435,13 +441,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.admin_variables.checksum, row[3]); checksums_values.admin_variables.last_changed = now; checksums_values.admin_variables.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for admin_variables from peer %s:%d, version %llu, epoch %llu, checksum %s. Not syncing yet ...\n", hostname, port, checksums_values.admin_variables.version, checksums_values.admin_variables.epoch, checksums_values.admin_variables.checksum); proxy_info("Cluster: detected a new checksum for admin_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.admin_variables.version, checksums_values.admin_variables.epoch, checksums_values.admin_variables.checksum); + if (strcmp(checksums_values.admin_variables.checksum, GloVars.checksums_values.admin_variables.checksum) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for admin_variables from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.admin_variables.checksum); + proxy_info("Cluster: checksum for admin_variables from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.admin_variables.checksum); + } } else { - proxy_info("Cluster: checksum for admin_variables from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.admin_variables.checksum); checksums_values.admin_variables.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for admin_variables from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.admin_variables.version, checksums_values.admin_variables.epoch, + checksums_values.admin_variables.checksum, GloVars.checksums_values.admin_variables.checksum, checksums_values.admin_variables.diff_check); } if (strcmp(checksums_values.admin_variables.checksum, GloVars.checksums_values.admin_variables.checksum) == 0) { checksums_values.admin_variables.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for admin_variables from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.admin_variables.checksum); } continue; } @@ -453,15 +466,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.mysql_query_rules.checksum, row[3]); checksums_values.mysql_query_rules.last_changed = now; checksums_values.mysql_query_rules.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for mysql_query_rules from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_query_rules.version, checksums_values.mysql_query_rules.epoch, checksums_values.mysql_query_rules.checksum); proxy_info("Cluster: detected a new checksum for mysql_query_rules from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_query_rules.version, checksums_values.mysql_query_rules.epoch, checksums_values.mysql_query_rules.checksum); if (strcmp(checksums_values.mysql_query_rules.checksum, GloVars.checksums_values.mysql_query_rules.checksum) == 0) { - proxy_info("Cluster: checksum for mysql_query_rules from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_query_rules.checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_query_rules from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_query_rules.checksum); + proxy_info("Cluster: checksum for mysql_query_rules from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_query_rules.checksum); } } else { checksums_values.mysql_query_rules.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_query_rules from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_query_rules.version, checksums_values.mysql_query_rules.epoch, + checksums_values.mysql_query_rules.checksum, GloVars.checksums_values.mysql_query_rules.checksum, checksums_values.mysql_query_rules.diff_check); } if (strcmp(checksums_values.mysql_query_rules.checksum, GloVars.checksums_values.mysql_query_rules.checksum) == 0) { checksums_values.mysql_query_rules.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_query_rules from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_query_rules.checksum); } continue; } @@ -473,15 +491,46 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.mysql_servers.checksum, row[3]); checksums_values.mysql_servers.last_changed = now; checksums_values.mysql_servers.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for mysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers.version, checksums_values.mysql_servers.epoch, checksums_values.mysql_servers.checksum); proxy_info("Cluster: detected a new checksum for mysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers.version, checksums_values.mysql_servers.epoch, checksums_values.mysql_servers.checksum); if (strcmp(checksums_values.mysql_servers.checksum, GloVars.checksums_values.mysql_servers.checksum) == 0) { - proxy_info("Cluster: checksum for mysql_servers from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers.checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers.checksum); + proxy_info("Cluster: checksum for mysql_servers from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers.checksum); } } else { checksums_values.mysql_servers.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_servers.version, checksums_values.mysql_servers.epoch, + checksums_values.mysql_servers.checksum, GloVars.checksums_values.mysql_servers.checksum, checksums_values.mysql_servers.diff_check); } if (strcmp(checksums_values.mysql_servers.checksum, GloVars.checksums_values.mysql_servers.checksum) == 0) { checksums_values.mysql_servers.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_servers.checksum); + } + continue; + } + if (strcmp(row[0], "mysql_servers_incoming")==0) { + checksums_values.mysql_servers_incoming.version = atoll(row[1]); + checksums_values.mysql_servers_incoming.epoch = atoll(row[2]); + checksums_values.mysql_servers_incoming.last_updated = now; + if (strcmp(checksums_values.mysql_servers_incoming.checksum, row[3])) { + strcpy(checksums_values.mysql_servers_incoming.checksum, row[3]); + checksums_values.mysql_servers_incoming.last_changed = now; + checksums_values.mysql_servers_incoming.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for mysql_servers_incoming from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers_incoming.version, checksums_values.mysql_servers_incoming.epoch, checksums_values.mysql_servers_incoming.checksum); + proxy_info("Cluster: detected a new checksum for mysql_servers_incoming from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers_incoming.version, checksums_values.mysql_servers_incoming.epoch, checksums_values.mysql_servers_incoming.checksum); + if (strcmp(checksums_values.mysql_servers_incoming.checksum, GloVars.checksums_values.mysql_servers_incoming.checksum) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); + proxy_info("Cluster: checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); + } + } + else { + checksums_values.mysql_servers_incoming.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_servers_incoming.version, checksums_values.mysql_servers_incoming.epoch, + checksums_values.mysql_servers_incoming.checksum, GloVars.checksums_values.mysql_servers_incoming.checksum, checksums_values.mysql_servers_incoming.diff_check); + } + if (strcmp(checksums_values.mysql_servers_incoming.checksum, GloVars.checksums_values.mysql_servers_incoming.checksum) == 0) { + checksums_values.mysql_servers_incoming.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); } continue; } @@ -493,15 +542,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.mysql_users.checksum, row[3]); checksums_values.mysql_users.last_changed = now; checksums_values.mysql_users.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for mysql_users from peer %s:%d, version %llu, epoch %llu, checksum %s. Not syncing yet ...\n", hostname, port, checksums_values.mysql_users.version, checksums_values.mysql_users.epoch, checksums_values.mysql_users.checksum); proxy_info("Cluster: detected a new checksum for mysql_users from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_users.version, checksums_values.mysql_users.epoch, checksums_values.mysql_users.checksum); if (strcmp(checksums_values.mysql_users.checksum, GloVars.checksums_values.mysql_users.checksum) == 0) { - proxy_info("Cluster: checksum for mysql_users from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_users.checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_users from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_users.checksum); + proxy_info("Cluster: checksum for mysql_users from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_users.checksum); } } else { checksums_values.mysql_users.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_users from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_users.version, checksums_values.mysql_users.epoch, + checksums_values.mysql_users.checksum, GloVars.checksums_values.mysql_users.checksum, checksums_values.mysql_users.diff_check); } if (strcmp(checksums_values.mysql_users.checksum, GloVars.checksums_values.mysql_users.checksum) == 0) { checksums_values.mysql_users.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_users from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_users.checksum); } continue; } @@ -513,15 +567,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.mysql_variables.checksum, row[3]); checksums_values.mysql_variables.last_changed = now; checksums_values.mysql_variables.diff_check = 1; - proxy_info("Cluster: detected a new checksum for mysql_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_variables.version, checksums_values.mysql_variables.epoch, checksums_values.mysql_variables.checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for mysql_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_variables.version, checksums_values.mysql_variables.epoch, checksums_values.mysql_variables.checksum); + proxy_info("Cluster: detected a new checksum for mysql_variables from peer %s:%d, version %llu, epoch %llu, checksum %s. Not syncing yet ...\n", hostname, port, checksums_values.mysql_variables.version, checksums_values.mysql_variables.epoch, checksums_values.mysql_variables.checksum); if (strcmp(checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.checksum) == 0) { - proxy_info("Cluster: checksum for mysql_variables from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_variables.checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_variables from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_variables.checksum); + proxy_info("Cluster: checksum for mysql_variables from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_variables.checksum); } } else { checksums_values.mysql_variables.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_variables from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_variables.version, checksums_values.mysql_variables.epoch, + checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.checksum, checksums_values.mysql_variables.diff_check); } if (strcmp(checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.checksum) == 0) { checksums_values.mysql_variables.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_variables from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_variables.checksum); } continue; } @@ -533,15 +592,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.proxysql_servers.checksum, row[3]); checksums_values.proxysql_servers.last_changed = now; checksums_values.proxysql_servers.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a new checksum for proxysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.proxysql_servers.version, checksums_values.proxysql_servers.epoch, checksums_values.proxysql_servers.checksum); proxy_info("Cluster: detected a new checksum for proxysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.proxysql_servers.version, checksums_values.proxysql_servers.epoch, checksums_values.proxysql_servers.checksum); if (strcmp(checksums_values.proxysql_servers.checksum, GloVars.checksums_values.proxysql_servers.checksum) == 0) { - proxy_info("Cluster: checksum for proxysql_servers from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.proxysql_servers.checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for proxysql_servers from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.proxysql_servers.checksum); + proxy_info("Cluster: checksum for proxysql_servers from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.proxysql_servers.checksum); } } else { checksums_values.proxysql_servers.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for proxysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.proxysql_servers.version, checksums_values.proxysql_servers.epoch, + checksums_values.proxysql_servers.checksum, GloVars.checksums_values.proxysql_servers.checksum, checksums_values.proxysql_servers.diff_check); } if (strcmp(checksums_values.proxysql_servers.checksum, GloVars.checksums_values.proxysql_servers.checksum) == 0) { checksums_values.proxysql_servers.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for proxysql_servers from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.proxysql_servers.checksum); } continue; } @@ -553,15 +617,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.ldap_variables.checksum, row[3]); checksums_values.ldap_variables.last_changed = now; checksums_values.ldap_variables.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for ldap_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.ldap_variables.version, checksums_values.ldap_variables.epoch, checksums_values.ldap_variables.checksum); proxy_info("Cluster: detected a new checksum for ldap_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.ldap_variables.version, checksums_values.ldap_variables.epoch, checksums_values.ldap_variables.checksum); if (strcmp(checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.checksum) == 0) { - proxy_info("Cluster: checksum for ldap_variables from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.ldap_variables.checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for ldap_variables from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.ldap_variables.checksum); + proxy_info("Cluster: checksum for ldap_variables from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.ldap_variables.checksum); } } else { checksums_values.ldap_variables.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for ldap_variables from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.ldap_variables.version, checksums_values.ldap_variables.epoch, + checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.checksum, checksums_values.ldap_variables.diff_check); } if (strcmp(checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.checksum) == 0) { checksums_values.ldap_variables.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for ldap_variables from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.ldap_variables.checksum); } continue; } @@ -587,6 +656,13 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (strcmp(v->checksum, GloVars.checksums_values.mysql_servers.checksum) == 0) { v->diff_check = 0; } + if (v->diff_check) + v->diff_check++; + v = &checksums_values.mysql_servers_incoming; + v->last_updated = now; + if (strcmp(v->checksum, GloVars.checksums_values.mysql_servers_incoming.checksum) == 0) { + v->diff_check = 0; + } if (v->diff_check) v->diff_check++; v = &checksums_values.mysql_users; @@ -644,49 +720,113 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (v->epoch > own_epoch) // epoch is newer ) { if (v->diff_check >= diff_mqr) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_mysql_query_rules_from_peer(v_exp_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mqr*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mqr * 10)); proxy_error("Cluster: detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mqr*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_query_rules_share_epoch]->Increment(); } } else { if (v->diff_check && (v->diff_check % (diff_mqr*10)) == 0) { - proxy_warning("Cluster: detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL QUERY RULES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mqr*10)); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL QUERY RULES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mqr * 10)); + proxy_warning("Cluster: detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL QUERY RULES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mqr*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_mysql_query_rules_version_one]->Increment(); } } } if (diff_ms) { - v = &checksums_values.mysql_servers; - unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.version,0); - unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.epoch,0); - char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.checksum,0); - const std::string v_exp_checksum { v->checksum }; + mysql_servers_sync_algorithm mysql_server_sync_algo = (mysql_servers_sync_algorithm)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_sync_algorithm, 0); + + if (mysql_server_sync_algo == mysql_servers_sync_algorithm::auto_select) { + mysql_server_sync_algo = (GloVars.global.monitor == false) ? + mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_incoming : mysql_servers_sync_algorithm::mysql_servers_incoming; + } + + v = &checksums_values.mysql_servers_incoming; + const unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_incoming.version, 0); + const unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_incoming.epoch, 0); + const char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_incoming.checksum, 0); + bool runtime_mysql_servers_already_loaded = false; if (v->version > 1) { if ( (own_version == 1) // we just booted || (v->epoch > own_epoch) // epoch is newer - ) { + ) { if (v->diff_check >= diff_ms) { - proxy_info("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_mysql_servers_from_peer(v_exp_checksum, v->epoch); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + + if (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_incoming) { + ProxySQL_Checksum_Value_2* runtime_mysql_server_checksum = &checksums_values.mysql_servers; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_incoming and mysql_servers from peer %s:%d", hostname, port); + // fetching mysql server incoming and runtime mysql server + GloProxyCluster->pull_mysql_servers_incoming_from_peer({ v->checksum, v->epoch }, + { runtime_mysql_server_checksum->checksum, runtime_mysql_server_checksum->epoch }, true); + + runtime_mysql_servers_already_loaded = true; + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_incoming from peer %s:%d", hostname, port); + // fetching mysql server incoming + // to avoid runtime_mysql_servers generating new epoch, we are passing mysql_server_incoming checksum and epoch + GloProxyCluster->pull_mysql_servers_incoming_from_peer({ v->checksum, v->epoch }, { v->checksum, v->epoch }); + } } } - if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms*10)) == 0)) { - proxy_error("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms*10)); + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms * 10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); + proxy_error("Cluster: detected a peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_servers_share_epoch]->Increment(); } - } else { - if (v->diff_check && (v->diff_check % (diff_ms*10)) == 0) { - proxy_warning("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms*10)); + } + else { + if (v->diff_check && (v->diff_check % (diff_ms * 10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); + proxy_warning("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_mysql_servers_version_one]->Increment(); } } + + if (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_incoming && + runtime_mysql_servers_already_loaded == false) { + v = &checksums_values.mysql_servers; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.version, 0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.epoch, 0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.checksum, 0); + + if (v->version > 1) { + if ( + (own_version == 1) // we just booted + || + (v->epoch > own_epoch) // epoch is newer + //|| + //(v->checksum != own_checksum) // as runtime mysql server is generated on node itself, epoch can be newer so we also check checksum + ) { + if (v->diff_check >= diff_ms) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_runtime_mysql_servers_from_peer({ v->checksum, v->epoch }); + } + } + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms * 10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); + proxy_error("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_servers_share_epoch]->Increment(); + } + } + else { + if (v->diff_check && (v->diff_check % (diff_ms * 10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); + proxy_warning("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_mysql_servers_version_one]->Increment(); + } + } + } } if (diff_mu) { v = &checksums_values.mysql_users; @@ -702,17 +842,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (v->epoch > own_epoch) // epoch is newer ) { if (v->diff_check >= diff_mu) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_mysql_users_from_peer(v_exp_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mu*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mu * 10)); proxy_error("Cluster: detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mu*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_users_share_epoch]->Increment(); } } else { if (v->diff_check && (v->diff_check % (diff_mu*10)) == 0) { - proxy_warning("Cluster: detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL USERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mu*10)); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL USERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mu * 10)); + proxy_warning("Cluster: detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL USERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mu*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_mysql_users_version_one]->Increment(); } } @@ -731,16 +874,19 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (v->epoch > own_epoch) // epoch is newer ) { if (v->diff_check >= diff_mv) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_global_variables_from_peer("mysql", expected_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mv*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mv * 10)); proxy_error("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mv*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_variables_share_epoch]->Increment(); } } else { if (v->diff_check && (v->diff_check % (diff_mv*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv * 10)); proxy_warning("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_mysql_variables_version_one]->Increment(); } @@ -760,16 +906,19 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (v->epoch > own_epoch) // epoch is newer ) { if (v->diff_check >= diff_av) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_global_variables_from_peer("admin", expected_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_av*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av * 10)); proxy_error("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_admin_variables_share_epoch]->Increment(); } } else { if (v->diff_check && (v->diff_check % (diff_av*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av * 10)); proxy_warning("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_admin_variables_version_one]->Increment(); } @@ -789,16 +938,19 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (v->epoch > own_epoch) // epoch is newer ) { if (v->diff_check >= diff_lv) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_global_variables_from_peer("ldap", expected_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_lv*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_lv * 10)); proxy_error("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_lv*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_ldap_variables_share_epoch]->Increment(); } } else { if (v->diff_check && (v->diff_check % (diff_lv*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_lv * 10)); proxy_warning("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_lv*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_ldap_variables_version_one]->Increment(); } @@ -824,17 +976,20 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (v->epoch > own_epoch) // epoch is newer ) { if (v->diff_check >= diff_ps) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_proxysql_servers_from_peer(v_exp_checksum, v->epoch); } } if ((v_epoch == own_epoch) && v_diff_check && ((v_diff_check % (diff_ps*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps * 10)); proxy_error("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_proxysql_servers_share_epoch]->Increment(); } } else { if (v->diff_check && (v->diff_check % (diff_ps*10)) == 0) { - proxy_warning("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps*10)); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps * 10)); + proxy_warning("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_proxysql_servers_version_one]->Increment(); } } @@ -915,6 +1070,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Query Rules from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { @@ -929,6 +1085,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c rc_query = mysql_query(conn,CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING); if ( rc_query == 0) { result2 = mysql_store_result(conn); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Query Rules from peer %s:%d completed\n", hostname, port); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d completed\n", hostname, port); std::unique_ptr SQLite3_query_rules_resultset { get_SQLite3_resulset(result1) }; @@ -937,10 +1094,11 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c const uint64_t query_rules_hash = SQLite3_query_rules_resultset->raw_checksum() + SQLite3_query_rules_fast_routing_resultset->raw_checksum(); const string computed_checksum { get_checksum_from_hash(query_rules_hash) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Query Rules from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Query Rules from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); if (expected_checksum == computed_checksum) { - + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime MySQL Query Rules from peer %s:%d\n", hostname, port); proxy_info("Cluster: Loading to runtime MySQL Query Rules from peer %s:%d\n", hostname, port); pthread_mutex_lock(&GloAdmin->sql_query_global_mutex); //GloAdmin->admindb->execute("PRAGMA quick_check"); @@ -1039,21 +1197,27 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c } //GloAdmin->admindb->execute("PRAGMA integrity_check"); GloAdmin->admindb->execute("COMMIT"); + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading MySQL Query Rules to Runtime from peer %s:%d\n", hostname, port); // We release the ownership of the memory for 'SQLite3' resultsets here since now it's no longer // our responsability to free the memory, they should be directly passed to the 'Query Processor' GloAdmin->load_mysql_query_rules_to_runtime( SQLite3_query_rules_resultset.release(), SQLite3_query_rules_fast_routing_resultset.release(), expected_checksum, epoch ); if (GloProxyCluster->cluster_mysql_query_rules_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); GloAdmin->flush_GENERIC__from_to("mysql_query_rules", "memory_to_disk"); } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "NOT saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); proxy_info("Cluster: NOT saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); } pthread_mutex_unlock(&GloAdmin->sql_query_global_mutex); metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_success]->Increment(); } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Query Rules from peer %s:%d failed because of mismatching checksum. Expected: %s , Computed: %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str()); proxy_info( "Cluster: Fetching MySQL Query Rules from peer %s:%d failed because of mismatching checksum. Expected: %s , Computed: %s\n", hostname, port, expected_checksum.c_str(), computed_checksum.c_str() @@ -1061,10 +1225,12 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); } } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); } } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); } @@ -1075,6 +1241,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c mysql_free_result(result2); } } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); } @@ -1186,14 +1353,17 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Users from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); proxy_info("Cluster: Fetching MySQL Users from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, username, password, NULL, port, NULL, 0); if (rc_conn == nullptr) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); if (GloMyLdapAuth) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); } @@ -1208,9 +1378,11 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu MYSQL_RES* mysql_users_result = mysql_store_result(conn); MYSQL_RES* ldap_mapping_result = nullptr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Users from peer %s:%d completed\n", hostname, port); proxy_info("Cluster: Fetching MySQL Users from peer %s:%d completed\n", hostname, port); if (GloMyLdapAuth) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching LDAP Mappings from peer %s:%d.\n", hostname, port); proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d.\n", hostname, port); rc_query = mysql_query( @@ -1219,8 +1391,10 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu if (rc_query == 0) { ldap_mapping_result = mysql_store_result(conn); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching LDAP Mappings from peer %s:%d completed\n", hostname, port); proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d completed\n", hostname, port); } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); } @@ -1230,6 +1404,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu const uint64_t users_raw_checksum = get_mysql_users_checksum(mysql_users_result, ldap_mapping_result, mysql_users_resultset); const string computed_checksum { get_checksum_from_hash(users_raw_checksum) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Users from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Users from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); if (expected_checksum == computed_checksum) { @@ -1241,22 +1416,28 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu mysql_free_result(ldap_mapping_result); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime MySQL Users from peer %s:%d\n", hostname, port); proxy_info("Cluster: Loading to runtime MySQL Users from peer %s:%d\n", hostname, port); if (GloMyLdapAuth) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime LDAP Mappings from peer %s:%d\n", hostname, port); proxy_info("Cluster: Loading to runtime LDAP Mappings from peer %s:%d\n", hostname, port); } GloAdmin->init_users(std::move(mysql_users_resultset), expected_checksum, epoch); if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Users from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk MySQL Users from peer %s:%d\n", hostname, port); if (GloMyLdapAuth) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk LDAP Mappings from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk LDAP Mappings from peer %s:%d\n", hostname, port); } GloAdmin->flush_mysql_users__from_memory_to_disk(); } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "NOT saving to disk MySQL Users from peer %s:%d\n", hostname, port); proxy_info("Cluster: NOT saving to disk MySQL Users from peer %s:%d\n", hostname, port); if (GloMyLdapAuth) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "NOT Saving to disk LDAP Mappings from peer %s:%d\n", hostname, port); proxy_info("Cluster: NOT Saving to disk LDAP Mappings from peer %s:%d\n", hostname, port); } } @@ -1274,6 +1455,8 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu mysql_free_result(ldap_mapping_result); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Users from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str()); proxy_info( "Cluster: Fetching MySQL Users from peer %s:%d failed: Checksum changed from %s to %s\n", hostname, port, expected_checksum.c_str(), computed_checksum.c_str() @@ -1285,10 +1468,12 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu } } } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); if (GloMyLdapAuth) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); } @@ -1366,7 +1551,7 @@ int ProxySQL_Cluster::fetch_and_store(MYSQL* conn, const fetch_query& f_query, M } /** - * @brief Generates a hash for the resulset received for the query 'CLUSTER_QUERY_MYSQL_SERVERS'. + * @brief Generates a hash for the resulset received for the query 'CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS'. * @details Remember that this query query is intercepted by 'ProxySQL_Admin' and always answered via * 'MySQL_HostGroups_Manager::dump_table_proxysql_servers'. * @param resultset The resulset resulting from the mentioned query. @@ -1426,7 +1611,7 @@ uint64_t mysql_servers_raw_checksum(MYSQL_RES* resultset) { /** * @brief Generates a hash from the received resultsets from executing the following queries in the specified * order: - * - CLUSTER_QUERY_MYSQL_SERVERS. + * - CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS. * - CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS. * - CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS. * - CLUSTER_QUERY_MYSQL_GALERA. @@ -1438,11 +1623,11 @@ uint64_t mysql_servers_raw_checksum(MYSQL_RES* resultset) { * @param results The resultsets from whose to compute the checksum. Previous described order is required. * @return Zero if the received resultset were empty, the computed hash otherwise. */ -uint64_t compute_servers_tables_raw_checksum(const vector& results) { +uint64_t compute_servers_tables_raw_checksum(const vector& results, size_t size) { bool init = false; SpookyHash myhash {}; - for (size_t i = 0; i < results.size(); i++) { + for (size_t i = 0; i < size; i++) { uint64_t raw_hash = 0; if (i == 0) { @@ -1469,7 +1654,7 @@ uint64_t compute_servers_tables_raw_checksum(const vector& results) return servers_hash; } -incoming_servers_t convert_servers_resultsets(const std::vector& results) { +incoming_servers_t convert_mysql_servers_resultsets(const std::vector& results) { if (results.size() != sizeof(incoming_servers_t) / sizeof(void*)) { return incoming_servers_t {}; } else { @@ -1479,18 +1664,25 @@ incoming_servers_t convert_servers_resultsets(const std::vector& res get_SQLite3_resulset(results[2]).release(), get_SQLite3_resulset(results[3]).release(), get_SQLite3_resulset(results[4]).release(), - get_SQLite3_resulset(results[5]).release() + get_SQLite3_resulset(results[5]).release(), + get_SQLite3_resulset(results[6]).release(), }; } } -void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, const time_t epoch) { +/** + * @brief mysql_servers records will be fetched from remote peer and saved locally. + * + * @param peer_runtime_mysql_server checksum and epoch of mysql_servers from remote peer + */ +void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server) { char * hostname = NULL; char * ip_address = NULL; uint16_t port = 0; char * peer_checksum = NULL; - pthread_mutex_lock(&GloProxyCluster->update_mysql_servers_mutex); - nodes.get_peer_to_sync_mysql_servers(&hostname, &port, &peer_checksum, &ip_address); + + pthread_mutex_lock(&GloProxyCluster->update_runtime_mysql_servers_mutex); + nodes.get_peer_to_sync_runtime_mysql_servers(&hostname, &port, &peer_checksum, &ip_address); if (hostname) { char *username = NULL; char *password = NULL; @@ -1502,6 +1694,120 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, goto __exit_pull_mysql_servers_from_peer; } GloProxyCluster->get_credentials(&username, &password); + if (strlen(username)) { // do not monitor if the username is empty + unsigned int timeout = 1; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + + { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching 'MySQL Servers' from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_checksum); + proxy_info("Cluster: Fetching 'MySQL Servers' from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_checksum); + rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, username, password, NULL, port, NULL, 0); + if (rc_conn) { + MySQL_Monitor::dns_cache_update_socket(conn->host, conn->net.fd); + + // servers messages + std::string fetch_servers_done; + string_format("Cluster: Fetching 'MySQL Servers' from peer %s:%d completed\n", fetch_servers_done, hostname, port); + std::string fetch_servers_err; + string_format("Cluster: Fetching 'MySQL Servers' from peer %s:%d failed: \n", fetch_servers_err, hostname, port); + + // Create fetching query + fetch_query query = { + CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS, + p_cluster_counter::pulled_mysql_servers_success, + p_cluster_counter::pulled_mysql_servers_failure, + { "", fetch_servers_done, fetch_servers_err } + }; + + MYSQL_RES* result = nullptr; + + if (fetch_and_store(conn, query, &result) != 0) { + if (result) { + mysql_free_result(result); + result = nullptr; + } + } + + if (result != nullptr) { + const uint64_t servers_hash = mysql_servers_raw_checksum(result); + const string computed_checksum{ get_checksum_from_hash(servers_hash) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + + if (computed_checksum == peer_checksum) { + MyHGM->wrlock(); + std::unique_ptr runtime_mysql_servers_resultset = get_SQLite3_resulset(result); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading runtime_mysql_servers from peer %s:%d into mysql_servers_incoming", hostname, port); + MyHGM->servers_add(runtime_mysql_servers_resultset.get()); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Updating runtime_mysql_servers from peer %s:%d", hostname, port); + MyHGM->update_runtime_mysql_servers_table(runtime_mysql_servers_resultset.get(), peer_runtime_mysql_server); + MyHGM->wrunlock(); + + // free result + mysql_free_result(result); + + metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_success]->Increment(); + } + } + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: Fetching MySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_failure]->Increment(); + } + } + if (username) { + free(username); + } + if (password) { + free(password); + } +__exit_pull_mysql_servers_from_peer: + if (conn) { + if (conn->net.pvio) { + mysql_close(conn); + } + } + free(hostname); + + if (peer_checksum) + free(peer_checksum); + + if (ip_address) + free(ip_address); + } + pthread_mutex_unlock(&GloProxyCluster->update_runtime_mysql_servers_mutex); +} + +/** + * @brief mysql_servers_incoming (mysql_servers_v2) records will be fetched from remote peer. mysql_servers records will be + * fetched if fetch_runtime_mysql_servers flag is true. + * + * @param peer_mysql_server_incoming checksum and epoch of mysql_servers_incoming from remote peer + * @param peer_runtime_mysql_server checksum and epoch of mysql_servers from remote peer + * @param fetch_runtime_mysql_servers fetch mysql_servers records if value is true + */ +void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming, + const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, bool fetch_runtime_mysql_servers) { + char* hostname = NULL; + char* ip_address = NULL; + uint16_t port = 0; + char* peer_mysql_servers_incoming_checksum = NULL; + char* peer_runtime_mysql_servers_checksum = NULL; + + pthread_mutex_lock(&GloProxyCluster->update_mysql_servers_incoming_mutex); + nodes.get_peer_to_sync_mysql_servers_incoming(&hostname, &port, &peer_mysql_servers_incoming_checksum, + &peer_runtime_mysql_servers_checksum, &ip_address); + if (hostname) { + char* username = NULL; + char* password = NULL; + // bool rc_bool = true; + MYSQL* rc_conn; + MYSQL* conn = mysql_init(NULL); + if (conn == NULL) { + proxy_error("Unable to run mysql_init()\n"); + goto __exit_pull_mysql_servers_incoming_from_peer; + } + GloProxyCluster->get_credentials(&username, &password); if (strlen(username)) { // do not monitor if the username is empty unsigned int timeout = 1; // unsigned int timeout_long = 60; @@ -1509,18 +1815,19 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } - proxy_info("Cluster: Fetching MySQL Servers from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers Incoming from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_mysql_servers_incoming_checksum); + proxy_info("Cluster: Fetching MySQL Servers Incoming from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_mysql_servers_incoming_checksum); rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { MySQL_Monitor::dns_cache_update_socket(conn->host, conn->net.fd); - std::vector results {}; + std::vector results(7,nullptr); // servers messages std::string fetch_servers_done = ""; - string_format("Cluster: Fetching MySQL Servers from peer %s:%d completed\n", fetch_servers_done, hostname, port); + string_format("Cluster: Fetching 'MySQL Servers Incoming' from peer %s:%d completed\n", fetch_servers_done, hostname, port); std::string fetch_servers_err = ""; - string_format("Cluster: Fetching MySQL Servers from peer %s:%d failed: \n", fetch_servers_err, hostname, port); + string_format("Cluster: Fetching 'MySQL Servers Incoming' from peer %s:%d failed: \n", fetch_servers_err, hostname, port); // group_replication_hostgroups messages std::string fetch_group_replication_hostgroups = ""; @@ -1555,7 +1862,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, */ fetch_query queries[] = { { - CLUSTER_QUERY_MYSQL_SERVERS, + CLUSTER_QUERY_MYSQL_SERVERS_INCOMING, p_cluster_counter::pulled_mysql_servers_success, p_cluster_counter::pulled_mysql_servers_failure, { "", fetch_servers_done, fetch_servers_err } @@ -1597,40 +1904,78 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, int it_err = fetch_and_store(conn, queries[i], &fetch_res); if (it_err == 0) { - results.push_back(fetch_res); - } else { + results[i] = fetch_res; + } + else { fetching_error = true; break; } } + // fetch_runtime_mysql_servers value depends on 'cluster_mysql_servers_sync_algorithm' + if (fetch_runtime_mysql_servers == true) { + // Fetching runtime mysql servers (mysql_servers) configuration from remote peer + std::string fetch_runtime_servers_done = ""; + string_format("Cluster: Fetching 'MySQL Servers' from peer %s:%d completed\n", fetch_runtime_servers_done, hostname, port); + std::string fetch_runtime_servers_err = ""; + string_format("Cluster: Fetching 'MySQL Servers' from peer %s:%d failed: \n", fetch_runtime_servers_err, hostname, port); + + // Query definition used to fetch data from a peer. + fetch_query query = { + CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS, + p_cluster_counter::pulled_mysql_servers_success, + p_cluster_counter::pulled_mysql_servers_failure, + { "", fetch_runtime_servers_done, fetch_runtime_servers_err } + }; + + MYSQL_RES* fetch_res = nullptr; + if (fetch_and_store(conn, query, &fetch_res) == 0) { + results[6] = fetch_res; + }else { + fetching_error = true; + } + } + if (fetching_error == false) { - const uint64_t servers_hash = compute_servers_tables_raw_checksum(results); - const string computed_checksum { get_checksum_from_hash(servers_hash) }; - proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + const uint64_t servers_hash = compute_servers_tables_raw_checksum(results, 6); // ignore runtime_mysql_servers in checksum calculation + const string computed_checksum{ get_checksum_from_hash(servers_hash) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers Incoming from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + proxy_info("Cluster: Computed checksum for MySQL Servers Incoming from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + + bool runtime_checksum_matches = true; + + if (results[6]) { + const uint64_t runtime_mysql_server_hash = mysql_servers_raw_checksum(results[6]); + const std::string runtime_mysql_server_computed_checksum = get_checksum_from_hash(runtime_mysql_server_hash); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); + proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); + runtime_checksum_matches = (runtime_mysql_server_computed_checksum == peer_runtime_mysql_servers_checksum); + } - if (computed_checksum == peer_checksum) { + if (computed_checksum == peer_mysql_servers_incoming_checksum && runtime_checksum_matches == true) { // No need to perform the conversion if checksums don't match - const incoming_servers_t incoming_servers { convert_servers_resultsets(results) }; + const incoming_servers_t incoming_servers{ convert_mysql_servers_resultsets(results) }; // we are OK to sync! + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching checksum for 'MySQL Servers' from peer %s:%d successful. Checksum: %s\n", hostname, port, computed_checksum.c_str()); proxy_info("Cluster: Fetching checksum for 'MySQL Servers' from peer %s:%d successful. Checksum: %s\n", hostname, port, computed_checksum.c_str()); // sync mysql_servers + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_servers table\n"); proxy_info("Cluster: Writing mysql_servers table\n"); GloAdmin->mysql_servers_wrlock(); GloAdmin->admindb->execute("DELETE FROM mysql_servers"); MYSQL_ROW row; - char *q=(char *)"INSERT INTO mysql_servers (hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) VALUES (%s, \"%s\", %s, %s, \"%s\", %s, %s, %s, %s, %s, %s, '%s')"; + char* q = (char*)"INSERT INTO mysql_servers (hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) VALUES (%s, \"%s\", %s, %s, \"%s\", %s, %s, %s, %s, %s, %s, '%s')"; while ((row = mysql_fetch_row(results[0]))) { int i; - int l=0; - for (i=0; i<11; i++) { - l+=strlen(row[i]); + int l = 0; + for (i = 0; i < 11; i++) { + l += strlen(row[i]); } - char *o=escape_string_single_quotes(row[11],false); - char *query = (char *)malloc(strlen(q)+i+strlen(o)+64); + char* o = escape_string_single_quotes(row[11], false); + char* query = (char*)malloc(strlen(q) + i + strlen(o) + 64); - sprintf(query,q,row[0],row[1],row[2],row[3], ( strcmp(row[4],"SHUNNED")==0 ? "ONLINE" : row[4] ), row[5], row[6],row[7],row[8],row[9],row[10],o); - if (o!=row[11]) { // there was a copy + sprintf(query, q, row[0], row[1], row[2], row[3], (strcmp(row[4], "SHUNNED") == 0 ? "ONLINE" : row[4]), row[5], row[6], row[7], row[8], row[9], row[10], o); + if (o != row[11]) { // there was a copy free(o); } GloAdmin->admindb->execute(query); @@ -1638,19 +1983,20 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, } // sync mysql_replication_hostgroups + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_replication_hostgroups table\n"); proxy_info("Cluster: Writing mysql_replication_hostgroups table\n"); GloAdmin->admindb->execute("DELETE FROM mysql_replication_hostgroups"); - q=(char *)"INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type, comment) VALUES (%s, %s, '%s', '%s')"; + q = (char*)"INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type, comment) VALUES (%s, %s, '%s', '%s')"; while ((row = mysql_fetch_row(results[1]))) { int i; - int l=0; - for (i=0; i<3; i++) { - l+=strlen(row[i]); + int l = 0; + for (i = 0; i < 3; i++) { + l += strlen(row[i]); } - char *o=escape_string_single_quotes(row[3],false); - char *query = (char *)malloc(strlen(q)+i+strlen(o)+64); - sprintf(query,q,row[0],row[1],row[2],o); - if (o!=row[3]) { // there was a copy + char* o = escape_string_single_quotes(row[3], false); + char* query = (char*)malloc(strlen(q) + i + strlen(o) + 64); + sprintf(query, q, row[0], row[1], row[2], o); + if (o != row[3]) { // there was a copy free(o); } GloAdmin->admindb->execute(query); @@ -1658,15 +2004,16 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, } // sync mysql_group_replication_hostgroups + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_group_replication_hostgroups table\n"); proxy_info("Cluster: Writing mysql_group_replication_hostgroups table\n"); GloAdmin->admindb->execute("DELETE FROM mysql_group_replication_hostgroups"); - q=(char*)"INSERT INTO mysql_group_replication_hostgroups ( " + q = (char*)"INSERT INTO mysql_group_replication_hostgroups ( " "writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, " "max_writers, writer_is_also_reader, max_transactions_behind, comment) "; - char *error = NULL; + char* error = NULL; int cols = 0; int affected_rows = 0; - SQLite3_result *resultset = NULL; + SQLite3_result* resultset = NULL; while ((row = mysql_fetch_row(results[2]))) { int i; int l = 0; @@ -1680,32 +2027,35 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, if (row[8] != nullptr) { fqs += "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, '%s')"; o = escape_string_single_quotes(row[8], false); - query = (char *)malloc(strlen(fqs.c_str()) + i + strlen(o) + 64); + query = (char*)malloc(strlen(fqs.c_str()) + i + strlen(o) + 64); sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], o); // free in case of 'o' being a copy if (o != row[8]) { free(o); } - } else { + } + else { // In case of comment being null, placeholder must not have '' fqs += "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"; o = const_cast("NULL"); - query = (char *)malloc(strlen(fqs.c_str()) + strlen("NULL") + i + 64); + query = (char*)malloc(strlen(fqs.c_str()) + strlen("NULL") + i + 64); sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], o); } GloAdmin->admindb->execute(query); free(query); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_group_replication_hostgroups'\n"); proxy_info("Dumping fetched 'mysql_group_replication_hostgroups'\n"); - GloAdmin->admindb->execute_statement((char *)"SELECT * FROM mysql_group_replication_hostgroups", &error , &cols , &affected_rows , &resultset); + GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_group_replication_hostgroups", &error, &cols, &affected_rows, &resultset); resultset->dump_to_stderr(); delete resultset; // sync mysql_galera_hostgroups + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_galera_hostgroups table\n"); proxy_info("Cluster: Writing mysql_galera_hostgroups table\n"); GloAdmin->admindb->execute("DELETE FROM mysql_galera_hostgroups"); - q=(char *)"INSERT INTO mysql_galera_hostgroups ( " + q = (char*)"INSERT INTO mysql_galera_hostgroups ( " "writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, " "max_writers, writer_is_also_reader, max_transactions_behind, comment) "; while ((row = mysql_fetch_row(results[3]))) { @@ -1721,32 +2071,35 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, if (row[8] != nullptr) { fqs += "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, '%s')"; o = escape_string_single_quotes(row[8], false); - query = (char *)malloc(strlen(fqs.c_str()) + i + strlen(o) + 64); + query = (char*)malloc(strlen(fqs.c_str()) + i + strlen(o) + 64); sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], o); // free in case of 'o' being a copy if (o != row[8]) { free(o); } - } else { + } + else { // In case of comment being null, placeholder must not have '' fqs += "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"; o = const_cast("NULL"); - query = (char *)malloc(strlen(fqs.c_str()) + i + strlen("NULL") + 64); + query = (char*)malloc(strlen(fqs.c_str()) + i + strlen("NULL") + 64); sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], o); } GloAdmin->admindb->execute(query); free(query); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_galera_hostgroups'\n"); proxy_info("Dumping fetched 'mysql_galera_hostgroups'\n"); - GloAdmin->admindb->execute_statement((char *)"SELECT * FROM mysql_galera_hostgroups", &error , &cols , &affected_rows , &resultset); + GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_galera_hostgroups", &error, &cols, &affected_rows, &resultset); resultset->dump_to_stderr(); delete resultset; // sync mysql_aws_aurora_hostgroups + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_aws_aurora_hostgroups table\n"); proxy_info("Cluster: Writing mysql_aws_aurora_hostgroups table\n"); GloAdmin->admindb->execute("DELETE FROM mysql_aws_aurora_hostgroups"); - q=(char *)"INSERT INTO mysql_aws_aurora_hostgroups ( " + q = (char*)"INSERT INTO mysql_aws_aurora_hostgroups ( " "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) "; while ((row = mysql_fetch_row(results[4]))) { @@ -1762,77 +2115,89 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, if (row[13] != nullptr) { fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, '%s')"; o = escape_string_single_quotes(row[13], false); - query = (char *)malloc(strlen(fqs.c_str()) + i + strlen(o) + 64); + query = (char*)malloc(strlen(fqs.c_str()) + i + strlen(o) + 64); sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], o); // free in case of 'o' being a copy if (o != row[13]) { free(o); } - } else { + } + else { // In case of comment being null, placeholder must not have '' fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s)"; o = const_cast("NULL"); - query = (char *)malloc(strlen(fqs.c_str()) + i + strlen("NULL") + 64); + query = (char*)malloc(strlen(fqs.c_str()) + i + strlen("NULL") + 64); sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], o); } GloAdmin->admindb->execute(query); free(query); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); proxy_info("Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); - GloAdmin->admindb->execute_statement((char *)"SELECT * FROM mysql_aws_aurora_hostgroups", &error , &cols , &affected_rows , &resultset); + GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_aws_aurora_hostgroups", &error, &cols, &affected_rows, &resultset); resultset->dump_to_stderr(); delete resultset; // sync mysql_hostgroup_attributes + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_hostgroup_attributes table\n"); proxy_info("Cluster: Writing mysql_hostgroup_attributes table\n"); GloAdmin->admindb->execute("DELETE FROM mysql_hostgroup_attributes"); { - const char * q=(const char *)"INSERT INTO mysql_hostgroup_attributes ( " + const char* q = (const char*)"INSERT INTO mysql_hostgroup_attributes ( " "hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, " "init_connect, multiplex, connection_warming, throttle_connections_per_sec, " "ignore_session_variables, comment) VALUES " "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; - sqlite3_stmt *statement1 = NULL; + sqlite3_stmt* statement1 = NULL; int rc = GloAdmin->admindb->prepare_v2(q, &statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); while ((row = mysql_fetch_row(results[5]))) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // hostgroup_id - rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_num_online_servers - rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atol(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // autocommit - rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atol(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // free_connections_pct - rc=(*proxy_sqlite3_bind_text)(statement1, 5, row[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // variable_name - rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // multiplex - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atol(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // connection_warming - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // throttle_connections_per_sec - rc=(*proxy_sqlite3_bind_text)(statement1, 9, row[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // ignore_session_variables - rc=(*proxy_sqlite3_bind_text)(statement1, 10, row[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // hostgroup_id + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_num_online_servers + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, atol(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // autocommit + rc = (*proxy_sqlite3_bind_int64)(statement1, 4, atol(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // free_connections_pct + rc = (*proxy_sqlite3_bind_text)(statement1, 5, row[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // variable_name + rc = (*proxy_sqlite3_bind_int64)(statement1, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // multiplex + rc = (*proxy_sqlite3_bind_int64)(statement1, 7, atol(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // connection_warming + rc = (*proxy_sqlite3_bind_int64)(statement1, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // throttle_connections_per_sec + rc = (*proxy_sqlite3_bind_text)(statement1, 9, row[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // ignore_session_variables + rc = (*proxy_sqlite3_bind_text)(statement1, 10, row[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); } (*proxy_sqlite3_finalize)(statement1); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_hostgroup_attributes'\n"); proxy_info("Dumping fetched 'mysql_hostgroup_attributes'\n"); - GloAdmin->admindb->execute_statement((char *)"SELECT * FROM mysql_hostgroup_attributes", &error , &cols , &affected_rows , &resultset); + GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_hostgroup_attributes", &error, &cols, &affected_rows, &resultset); resultset->dump_to_stderr(); delete resultset; - proxy_info("Cluster: Loading to runtime MySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->load_mysql_servers_to_runtime(incoming_servers, checksum, epoch); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime MySQL Servers Incoming from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Loading to runtime MySQL Servers Incoming from peer %s:%d\n", hostname, port); + GloAdmin->load_mysql_servers_to_runtime(incoming_servers, peer_runtime_mysql_server, peer_mysql_server_incoming); + if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { - proxy_info("Cluster: Saving to disk MySQL Servers from peer %s:%d\n", hostname, port); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Servers Incoming from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Saving to disk MySQL Servers Incoming from peer %s:%d\n", hostname, port); GloAdmin->flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); - } else { + } + else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); proxy_info("Cluster: Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); } GloAdmin->mysql_servers_wrunlock(); - } else { + } + else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, peer_mysql_servers_incoming_checksum, computed_checksum.c_str()); proxy_info( "Cluster: Fetching MySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", - hostname, port, peer_checksum, computed_checksum.c_str() + hostname, port, peer_mysql_servers_incoming_checksum, computed_checksum.c_str() ); metrics.p_counter_array[p_cluster_counter::pulled_mysql_variables_failure]->Increment(); } @@ -1844,7 +2209,9 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_success]->Increment(); } - } else { + } + else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching MySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_failure]->Increment(); } @@ -1855,7 +2222,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, if (password) { free(password); } -__exit_pull_mysql_servers_from_peer: + __exit_pull_mysql_servers_incoming_from_peer: if (conn) { if (conn->net.pvio) { mysql_close(conn); @@ -1865,8 +2232,14 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, if (ip_address) free(ip_address); + + if (peer_mysql_servers_incoming_checksum) + free (peer_mysql_servers_incoming_checksum); + + if (peer_runtime_mysql_servers_checksum) + free(peer_runtime_mysql_servers_checksum); } - pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_mutex); + pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_incoming_mutex); } void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, const string& expected_checksum, const time_t epoch) { @@ -1927,6 +2300,7 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching %s variables from peer %s:%d started\n", vars_type_str, hostname, port); proxy_info("Cluster: Fetching %s variables from peer %s:%d started\n", vars_type_str, hostname, port); rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, username, password, NULL, port, NULL, 0); @@ -1950,10 +2324,12 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c if (rc_query == 0) { MYSQL_RES *result = mysql_store_result(conn); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching %s Variables from peer %s:%d completed\n", vars_type_str, hostname, port); proxy_info("Cluster: Fetching %s Variables from peer %s:%d completed\n", vars_type_str, hostname, port); uint64_t glovars_hash = mysql_raw_checksum(result); string computed_checksum { get_checksum_from_hash(glovars_hash) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for %s Variables from peer %s:%d : %s\n", vars_type_str, hostname, port, computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for %s Variables from peer %s:%d : %s\n", vars_type_str, hostname, port, computed_checksum.c_str()); if (expected_checksum == computed_checksum) { @@ -1989,26 +2365,31 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c } mysql_free_result(result); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime %s Variables from peer %s:%d\n", vars_type_str, hostname, port); proxy_info("Cluster: Loading to runtime %s Variables from peer %s:%d\n", vars_type_str, hostname, port); if (var_type == "mysql") { GloAdmin->load_mysql_variables_to_runtime(expected_checksum, epoch); if (GloProxyCluster->cluster_mysql_variables_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Variables from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk MySQL Variables from peer %s:%d\n", hostname, port); GloAdmin->flush_mysql_variables__from_memory_to_disk(); } } else if (var_type == "admin") { - GloAdmin->load_admin_variables_to_runtime(expected_checksum, epoch); + GloAdmin->load_admin_variables_to_runtime(expected_checksum, epoch, false); if (GloProxyCluster->cluster_admin_variables_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk Admin Variables from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk Admin Variables from peer %s:%d\n", hostname, port); GloAdmin->flush_admin_variables__from_memory_to_disk(); } + } else if (var_type == "ldap") { GloAdmin->load_ldap_variables_to_runtime(expected_checksum, epoch); if (GloProxyCluster->cluster_ldap_variables_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk LDAP Variables from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk LDAP Variables from peer %s:%d\n", hostname, port); GloAdmin->flush_ldap_variables__from_memory_to_disk(); } @@ -2017,8 +2398,9 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c assert(0); } metrics.p_counter_array[success_metric]->Increment(); - } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching %s Variables from peer %s:%d failed: Checksum changed from %s to %s\n", + vars_type_str, hostname, port, expected_checksum.c_str(), computed_checksum.c_str()); proxy_info( "Cluster: Fetching %s Variables from peer %s:%d failed: Checksum changed from %s to %s\n", vars_type_str, hostname, port, expected_checksum.c_str(), computed_checksum.c_str() @@ -2026,10 +2408,12 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c metrics.p_counter_array[p_cluster_counter::pulled_mysql_variables_failure]->Increment(); } } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching %s Variables from peer %s:%d failed: %s\n", vars_type_str, hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching %s Variables from peer %s:%d failed: %s\n", vars_type_str, hostname, port, mysql_error(conn)); metrics.p_counter_array[failure_metric]->Increment(); } } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching %s Variables from peer %s:%d failed: %s\n", vars_type_str, hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching %s Variables from peer %s:%d failed: %s\n", vars_type_str, hostname, port, mysql_error(conn)); metrics.p_counter_array[failure_metric]->Increment(); } @@ -2079,6 +2463,8 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching ProxySQL Servers from peer %s:%d started. Expected checksum: %s\n", + hostname, port, expected_checksum.c_str()); proxy_info( "Cluster: Fetching ProxySQL Servers from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str() @@ -2092,6 +2478,7 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect MYSQL_RES* result = mysql_store_result(conn); uint64_t proxy_servers_hash = mysql_raw_checksum(result); const string computed_cks { get_checksum_from_hash(proxy_servers_hash) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching ProxySQL Servers from peer %s:%d completed. Computed checksum: %s\n", hostname, port, computed_cks.c_str()); proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d completed. Computed checksum: %s\n", hostname, port, computed_cks.c_str()); if (computed_cks == expected_checksum) { @@ -2114,6 +2501,7 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect free(query); } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'proxysql_servers'\n"); proxy_info("Dumping fetched 'proxysql_servers'\n"); char *error = NULL; int cols = 0; @@ -2123,16 +2511,21 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect resultset->dump_to_stderr(); delete resultset; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime ProxySQL Servers from peer %s:%d\n", hostname, port); proxy_info("Cluster: Loading to runtime ProxySQL Servers from peer %s:%d\n", hostname, port); GloAdmin->load_proxysql_servers_to_runtime(false, expected_checksum, epoch); if (GloProxyCluster->cluster_proxysql_servers_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); GloAdmin->flush_GENERIC__from_to("proxysql_servers","memory_to_disk"); } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "NOT saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); proxy_info("Cluster: NOT saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); } metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_success]->Increment(); } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching ProxySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_cks.c_str()); proxy_info( "Cluster: Fetching ProxySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", hostname, port, expected_checksum.c_str(), computed_cks.c_str() @@ -2141,10 +2534,12 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect } mysql_free_result(result); } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching ProxySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_failure]->Increment(); } } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching ProxySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_failure]->Increment(); } @@ -2403,6 +2798,9 @@ void ProxySQL_Cluster_Nodes::load_servers_list(SQLite3_result *resultset, bool _ node = new ProxySQL_Node_Entry(h_, p_, w_ , c_); node->set_active(true); umap_proxy_nodes.insert(std::make_pair(hash_, node)); + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Added new peer %s:%d\n", h_, p_); + ProxySQL_Node_Address * a = new ProxySQL_Node_Address(h_, p_, node->get_ipaddress()); pthread_attr_t attr; pthread_attr_init(&attr); @@ -2420,6 +2818,8 @@ void ProxySQL_Cluster_Nodes::load_servers_list(SQLite3_result *resultset, bool _ node->set_active(true); node->set_weight(w_); node->set_comment(c_); + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Peer %s:%d already exists. Updating it\n", h_, p_); } } remove_inactives(); @@ -2455,8 +2855,10 @@ bool ProxySQL_Cluster_Nodes::Update_Global_Checksum(char * _h, uint16_t _p, MYSQ while ((row = mysql_fetch_row(_r))) { unsigned long long v = atoll(row[0]); if (v == node->global_checksum) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum 0x%llX for peer %s:%d matches\n", v, node->get_hostname(), node->get_port()); ret = false; } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum for peer %s:%d is different from fetched one. Local checksum:[0x%llX] Fetched checksum:[0x%llX]\n", node->get_hostname(), node->get_port(), node->global_checksum, v); node->global_checksum = v; } } @@ -2542,11 +2944,12 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_query_rules(char **host, uin *host = hostname; *port = p; *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_query_rules version %llu, epoch %llu\n", hostname, p, version, epoch); proxy_info("Cluster: detected peer %s:%d with mysql_query_rules version %llu, epoch %llu\n", hostname, p, version, epoch); } } -void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address) { +void ProxySQL_Cluster_Nodes::get_peer_to_sync_runtime_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address) { unsigned long long version = 0; unsigned long long epoch = 0; unsigned long long max_epoch = 0; @@ -2608,8 +3011,88 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers(char **host, uint16_ *host = hostname; *port = p; *ip_address = ip_addr; - proxy_info("Cluster: detected peer %s:%d with mysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); *peer_checksum = pc; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, checksum %s\n", hostname, p, version, epoch, pc); + proxy_info("Cluster: detected peer %s:%d with mysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); + } +} + +void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_incoming(char** host, uint16_t* port, + char** peer_mysql_servers_incoming_checksum, char** peer_runtime_mysql_servers_checksum, char** ip_address) { + unsigned long long version = 0; + unsigned long long epoch = 0; + unsigned long long max_epoch = 0; + char* hostname = NULL; + char* ip_addr = NULL; + uint16_t p = 0; + char* mysql_servers_incoming_checksum = NULL; + char* runtime_mysql_servers_checksum = NULL; + //pthread_mutex_lock(&mutex); + //unsigned long long curtime = monotonic_time(); + unsigned int diff_ms = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, 0); + for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { + ProxySQL_Node_Entry* node = it->second; + ProxySQL_Checksum_Value_2* v = &node->checksums_values.mysql_servers_incoming; + if (v->version > 1) { + if (v->epoch > epoch) { + max_epoch = v->epoch; + if (v->diff_check > diff_ms) { + epoch = v->epoch; + version = v->version; + if (mysql_servers_incoming_checksum) { + free(mysql_servers_incoming_checksum); + } + if (runtime_mysql_servers_checksum) { + free(runtime_mysql_servers_checksum); + } + if (hostname) { + free(hostname); + } + if (ip_addr) { + free(ip_addr); + } + mysql_servers_incoming_checksum = strdup(v->checksum); + runtime_mysql_servers_checksum = strdup(node->checksums_values.mysql_servers.checksum); + hostname = strdup(node->get_hostname()); + const char* ip = node->get_ipaddress(); + if (ip) + ip_addr = strdup(ip); + p = node->get_port(); + } + } + } + it++; + } + // pthread_mutex_unlock(&mutex); + if (epoch) { + if (max_epoch > epoch) { + proxy_warning("Cluster: detected a peer with mysql_servers_incoming epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + if (hostname) { + free(hostname); + hostname = NULL; + } + if (mysql_servers_incoming_checksum) { + free(mysql_servers_incoming_checksum); + mysql_servers_incoming_checksum = NULL; + } + if (runtime_mysql_servers_checksum) { + free(runtime_mysql_servers_checksum); + runtime_mysql_servers_checksum = NULL; + } + if (ip_addr) { + free(ip_addr); + ip_addr = NULL; + } + } + } + if (hostname) { + *host = hostname; + *port = p; + *ip_address = ip_addr; + *peer_mysql_servers_incoming_checksum = mysql_servers_incoming_checksum; + *peer_runtime_mysql_servers_checksum = runtime_mysql_servers_checksum; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, mysql_servers_incoming checksum %s, runtime_mysql_servers %s\n", hostname, p, version, epoch, mysql_servers_incoming_checksum, runtime_mysql_servers_checksum); + proxy_info("Cluster: detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -2666,6 +3149,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_users(char **host, uint16_t *host = hostname; *port = p; *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_users version %llu, epoch %llu\n", hostname, p, version, epoch); proxy_info("Cluster: detected peer %s:%d with mysql_users version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -2720,6 +3204,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_variables(char **host, uint1 *host = hostname; *port = p; *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu\n", hostname, p, version, epoch); proxy_info("Cluster: detected peer %s:%d with mysql_variables version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -2775,6 +3260,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_admin_variables(char **host, uint1 *host = hostname; *port = p; *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu\n", hostname, p, version, epoch); proxy_info("Cluster: detected peer %s:%d with admin_variables version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -2829,6 +3315,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_ldap_variables(char **host, uint16 *host = hostname; *port = p; *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu\n", hostname, p, version, epoch); proxy_info("Cluster: detected peer %s:%d with ldap_variables version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -2886,6 +3373,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_proxysql_servers(char **host, uint *host = hostname; *port = p; *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); proxy_info("Cluster: detected peer %s:%d with proxysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -2909,14 +3397,15 @@ SQLite3_result * ProxySQL_Cluster_Nodes::stats_proxysql_servers_checksums() { //unsigned long long curtime = monotonic_time(); for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry * node = it->second; - ProxySQL_Checksum_Value_2 * vals[6]; + ProxySQL_Checksum_Value_2 * vals[7]; vals[0] = &node->checksums_values.admin_variables; vals[1] = &node->checksums_values.mysql_query_rules; vals[2] = &node->checksums_values.mysql_servers; vals[3] = &node->checksums_values.mysql_users; vals[4] = &node->checksums_values.mysql_variables; vals[5] = &node->checksums_values.proxysql_servers; - for (int i=0; i<6 ; i++) { + vals[6] = &node->checksums_values.mysql_servers_incoming; + for (int i=0; i<7 ; i++) { ProxySQL_Checksum_Value_2 *v = vals[i]; char **pta=(char **)malloc(sizeof(char *)*colnum); pta[0]=strdup(node->get_hostname()); @@ -2942,6 +3431,9 @@ SQLite3_result * ProxySQL_Cluster_Nodes::stats_proxysql_servers_checksums() { case 5: pta[2]=strdup((char *)"proxysql_servers"); break; + case 6: + pta[2] = strdup((char*)"mysql_servers_incoming"); + break; default: break; } @@ -3066,6 +3558,7 @@ vector> get_module_checksums(ProxySQL_N res.push_back({"mysql_users", &entry->checksums_values.mysql_users}); res.push_back({"mysql_variables", &entry->checksums_values.mysql_variables}); res.push_back({"proxysql_servers", &entry->checksums_values.proxysql_servers}); + res.push_back({"mysql_servers_incoming", &entry->checksums_values.mysql_servers_incoming}); return res; } @@ -3699,7 +4192,8 @@ cluster_metrics_map = std::make_tuple( ProxySQL_Cluster::ProxySQL_Cluster() : proxysql_servers_to_monitor(NULL) { pthread_mutex_init(&mutex,NULL); pthread_mutex_init(&update_mysql_query_rules_mutex,NULL); - pthread_mutex_init(&update_mysql_servers_mutex,NULL); + pthread_mutex_init(&update_runtime_mysql_servers_mutex,NULL); + pthread_mutex_init(&update_mysql_servers_incoming_mutex, NULL); pthread_mutex_init(&update_mysql_users_mutex,NULL); pthread_mutex_init(&update_proxysql_servers_mutex,NULL); pthread_mutex_init(&update_mysql_variables_mutex,NULL); @@ -3717,6 +4211,7 @@ ProxySQL_Cluster::ProxySQL_Cluster() : proxysql_servers_to_monitor(NULL) { cluster_mysql_servers_save_to_disk = true; cluster_mysql_users_save_to_disk = true; cluster_proxysql_servers_save_to_disk = true; + cluster_mysql_servers_sync_algorithm = 1; init_prometheus_counter_array(cluster_metrics_map, this->metrics.p_counter_array); init_prometheus_gauge_array(cluster_metrics_map, this->metrics.p_gauge_array); } @@ -3788,3 +4283,17 @@ void ProxySQL_Cluster::join_term_thread() { } pthread_mutex_unlock(&mutex); } + +void ProxySQL_Cluster_Nodes::Reset_Global_Checksums(bool lock) { + if (lock) { + pthread_mutex_lock(&mutex); + } + + for (auto& proxy_node_entry : umap_proxy_nodes) { + proxy_node_entry.second->global_checksum = 0; + } + + if (lock) { + pthread_mutex_unlock(&mutex); + } +} diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index 1df01f5313..65dd99907e 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -385,6 +385,11 @@ uint64_t ProxySQL_GlobalVariables::generate_global_checksum() { myhash.Update(v->checksum,strlen(v->checksum)); myhash.Update(&v->version,sizeof(v->version)); } + v = &checksums_values.mysql_servers_incoming; + if (v->version) { + myhash.Update(v->checksum, strlen(v->checksum)); + myhash.Update(&v->version, sizeof(v->version)); + } v = &checksums_values.mysql_users; if (v->version) { myhash.Update(v->checksum,strlen(v->checksum)); diff --git a/lib/debug.cpp b/lib/debug.cpp index 54007fd92b..917d0ec109 100644 --- a/lib/debug.cpp +++ b/lib/debug.cpp @@ -403,6 +403,7 @@ void init_debug_struct() { GloVars.global.gdbg_lvl[PROXY_DEBUG_QUERY_STATISTICS].name=(char *)"debug_query_statistics"; GloVars.global.gdbg_lvl[PROXY_DEBUG_RESTAPI].name=(char *)"debug_restapi"; GloVars.global.gdbg_lvl[PROXY_DEBUG_MONITOR].name=(char *)"debug_monitor"; + GloVars.global.gdbg_lvl[PROXY_DEBUG_CLUSTER].name=(char *)"debug_cluster"; for (i=0;i Date: Thu, 6 Apr 2023 12:37:25 +0500 Subject: [PATCH 02/56] Fixed code styling --- lib/MySQL_HostGroups_Manager.cpp | 39 +++++++++++--------------------- lib/ProxySQL_Cluster.cpp | 35 ++++++++++------------------ 2 files changed, 25 insertions(+), 49 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 8f7a28e535..50f39ff461 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1519,8 +1519,7 @@ void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); - } - else { + } else { if (GloMTH->variables.hostgroup_manager_verbose) { proxy_info("Dumping mysql_servers LEFT JOIN mysql_servers_incoming\n"); resultset->dump_to_stderr(); @@ -1550,8 +1549,7 @@ void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); - } - else { + } else { if (GloMTH->variables.hostgroup_manager_verbose) { proxy_info("Dumping mysql_servers JOIN mysql_servers_incoming\n"); @@ -1596,8 +1594,7 @@ void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); use_gtid = true; } - } - else { + } else { bool run_update = false; MySrvC* mysrvc = (MySrvC*)ptr; // carefully increase the 2nd index by 1 for every new column added @@ -1688,8 +1685,7 @@ void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result } if (use_gtid) { has_gtid_port = true; - } - else { + } else { has_gtid_port = false; } if (resultset) { delete resultset; resultset = NULL; } @@ -1717,8 +1713,7 @@ void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; - } - else { + } else { GloVars.checksums_values.mysql_servers.epoch = t; } @@ -1888,8 +1883,7 @@ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { std::unique_ptr server_mapping(new HostGroup_Server_Mapping(this)); fetched_server_mapping = server_mapping.get(); hostgroup_server_mapping.insert({ server_id, std::move(server_mapping) }); - } - else { + } else { fetched_server_mapping = itr->second.get(); } @@ -2190,8 +2184,7 @@ bool MySQL_HostGroups_Manager::commit( if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; - } - else { + } else { GloVars.checksums_values.mysql_servers.epoch = t; } @@ -2203,8 +2196,7 @@ bool MySQL_HostGroups_Manager::commit( if (peer_mysql_server_incoming.epoch != 0 && peer_mysql_server_incoming.checksum.empty() == false && GloVars.checksums_values.mysql_servers_incoming.checksum == peer_mysql_server_incoming.checksum) { GloVars.checksums_values.mysql_servers_incoming.epoch = peer_mysql_server_incoming.epoch; - } - else { + } else { GloVars.checksums_values.mysql_servers_incoming.epoch = t; } @@ -2275,8 +2267,7 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { rm_rows_count += 1; return true; - } - else { + } else { return false; } }; @@ -2287,8 +2278,7 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru resultset->rows_count = init_row_count - rm_rows_count; save_runtime_mysql_servers(resultset); - } - else { + } else { save_runtime_mysql_servers(runtime_mysql_servers); } @@ -2330,8 +2320,7 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_r if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { rm_rows_count += 1; return true; - } - else { + } else { return false; } }; @@ -2343,8 +2332,7 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_r save_mysql_servers_incoming(resultset); } - } - else { + } else { proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", (long unsigned int)0); } } else { @@ -2387,8 +2375,7 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_r if (strcasecmp(status, "ONLINE") == 0 || strcasecmp(status, "SHUNNED") == 0) { status_conv = "0"; - } - else if (strcasecmp(status, "OFFLINE_SOFT") == 0) { + } else if (strcasecmp(status, "OFFLINE_SOFT") == 0) { status_conv = "2"; } } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index c73f388215..292e8f6faf 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -350,8 +350,7 @@ ProxySQL_Node_Entry::ProxySQL_Node_Entry(char* _hostname, uint16_t _port, uint64 weight = _weight; if (_comment == NULL) { comment = strdup((char*)""); - } - else { + } else { comment = strdup(_comment); } @@ -522,8 +521,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); proxy_info("Cluster: checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); } - } - else { + } else { checksums_values.mysql_servers_incoming.diff_check++; proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_servers_incoming.version, checksums_values.mysql_servers_incoming.epoch, checksums_values.mysql_servers_incoming.checksum, GloVars.checksums_values.mysql_servers_incoming.checksum, checksums_values.mysql_servers_incoming.diff_check); @@ -783,8 +781,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { proxy_error("Cluster: detected a peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_servers_share_epoch]->Increment(); } - } - else { + } else { if (v->diff_check && (v->diff_check % (diff_ms * 10)) == 0) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); proxy_warning("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); @@ -818,8 +815,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { proxy_error("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_servers_share_epoch]->Increment(); } - } - else { + } else { if (v->diff_check && (v->diff_check % (diff_ms * 10)) == 0) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); proxy_warning("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms * 10)); @@ -1905,8 +1901,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers if (it_err == 0) { results[i] = fetch_res; - } - else { + } else { fetching_error = true; break; } @@ -1931,7 +1926,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers MYSQL_RES* fetch_res = nullptr; if (fetch_and_store(conn, query, &fetch_res) == 0) { results[6] = fetch_res; - }else { + } else { fetching_error = true; } } @@ -2033,8 +2028,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers if (o != row[8]) { free(o); } - } - else { + } else { // In case of comment being null, placeholder must not have '' fqs += "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"; o = const_cast("NULL"); @@ -2077,8 +2071,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers if (o != row[8]) { free(o); } - } - else { + } else { // In case of comment being null, placeholder must not have '' fqs += "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"; o = const_cast("NULL"); @@ -2121,8 +2114,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers if (o != row[13]) { free(o); } - } - else { + } else { // In case of comment being null, placeholder must not have '' fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s)"; o = const_cast("NULL"); @@ -2185,14 +2177,12 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Servers Incoming from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk MySQL Servers Incoming from peer %s:%d\n", hostname, port); GloAdmin->flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); - } - else { + } else { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); proxy_info("Cluster: Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); } GloAdmin->mysql_servers_wrunlock(); - } - else { + } else { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", hostname, port, peer_mysql_servers_incoming_checksum, computed_checksum.c_str()); proxy_info( @@ -2209,8 +2199,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_success]->Increment(); } - } - else { + } else { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); proxy_info("Cluster: Fetching MySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_failure]->Increment(); From 09f76511b1f2caa7f928ac2e6aa1d04b2e94aa19 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Thu, 6 Apr 2023 16:36:33 +0500 Subject: [PATCH 03/56] * Added runtime mysql server logic in commit. * Changed commit signature. * Few fixes --- include/MySQL_HostGroups_Manager.h | 13 +- lib/MySQL_HostGroups_Manager.cpp | 832 +++++++++++++++-------------- lib/ProxySQL_Admin.cpp | 2 +- lib/ProxySQL_Cluster.cpp | 4 +- 4 files changed, 438 insertions(+), 413 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 332fd161d6..ee313df312 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -393,14 +393,14 @@ class MySQL_HostGroups_Manager { #endif enum HGM_TABLES { - MYSQL_SERVERS = 0, + MYSQL_SERVERS_INCOMING = 0, MYSQL_REPLICATION_HOSTGROUPS, MYSQL_GROUP_REPLICATION_HOSTGROUPS, MYSQL_GALERA_HOSTGROUPS, MYSQL_AWS_AURORA_HOSTGROUPS, MYSQL_HOSTGROUP_ATTRIBUTES, + MYSQL_SERVERS, - MYSQL_SERVERS_INCOMING, __HGM_TABLES_SIZE }; @@ -698,9 +698,10 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); - void update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); - bool commit(SQLite3_result* runtime_mysql_servers = nullptr, SQLite3_result* mysql_servers_incoming = nullptr, - const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming = {}); + //void update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); + bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, + SQLite3_result* mysql_servers_incoming = nullptr, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming = {}, + bool only_commit_runtime_mysql_servers = false); void commit_update_checksums_from_tables(); void CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() @@ -826,6 +827,8 @@ class MySQL_HostGroups_Manager { MySrvC* find_server_in_hg(unsigned int _hid, const std::string& addr, int port); private: + static uint64_t compute_mysql_servers_raw_checksum(const SQLite3_result* runtime_mysql_servers); + void update_hostgroup_manager_mappings(); uint64_t get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers = nullptr); uint64_t get_mysql_servers_incoming_checksum(SQLite3_result* incoming_mysql_servers, bool use_precalculated_checksum = true); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 50f39ff461..3fb6fa1da0 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1495,235 +1495,234 @@ unsigned int MySQL_HostGroups_Manager::get_servers_table_version() { return __sync_fetch_and_add(&status.servers_table_version,0); } -/** - * @brief Generates runtime mysql server records and checksum. - * - * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in - * the specified order. - * @param can be null or previously generated runtime_mysql_servers resultset can be passed. - */ -void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, - const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server) { - char* error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - // if any server has gtid_port enabled, use_gtid is set to true - // and then has_gtid_port is set too - bool use_gtid = false; - - mydb->execute("DELETE FROM mysql_servers"); - generate_mysql_servers_table(); - - const char* query = "SELECT mem_pointer, t1.hostgroup_id, t1.hostname, t1.port FROM mysql_servers t1 LEFT OUTER JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE t2.hostgroup_id IS NULL"; - mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); - if (error) { - proxy_error("Error on %s : %s\n", query, error); - } else { - if (GloMTH->variables.hostgroup_manager_verbose) { - proxy_info("Dumping mysql_servers LEFT JOIN mysql_servers_incoming\n"); - resultset->dump_to_stderr(); - } - - for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { - SQLite3_row* r = *it; - long long ptr = atoll(r->fields[0]); - proxy_warning("Removed server at address %lld, hostgroup %s, address %s port %s. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", ptr, r->fields[1], r->fields[2], r->fields[3]); - MySrvC* mysrvc = (MySrvC*)ptr; - mysrvc->status = MYSQL_SERVER_STATUS_OFFLINE_HARD; - mysrvc->ConnectionsFree->drop_all_connections(); - char* q1 = (char*)"DELETE FROM mysql_servers WHERE mem_pointer=%lld"; - char* q2 = (char*)malloc(strlen(q1) + 32); - sprintf(q2, q1, ptr); - mydb->execute(q2); - free(q2); - } - } - if (resultset) { delete resultset; resultset = NULL; } - - mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); - - // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) - query = (char*)"SELECT t1.*, t2.gtid_port, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.gtid_port<>t2.gtid_port OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); - if (error) { - proxy_error("Error on %s : %s\n", query, error); - } else { - - if (GloMTH->variables.hostgroup_manager_verbose) { - proxy_info("Dumping mysql_servers JOIN mysql_servers_incoming\n"); - resultset->dump_to_stderr(); - } - // optimization #829 - int rc; - sqlite3_stmt* statement1 = NULL; - sqlite3_stmt* statement2 = NULL; - //sqlite3 *mydb3=mydb->get_db(); - char* query1 = (char*)"UPDATE mysql_servers SET mem_pointer = ?1 WHERE hostgroup_id = ?2 AND hostname = ?3 AND port = ?4"; - //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); - rc = mydb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, mydb); - char* query2 = (char*)"UPDATE mysql_servers SET weight = ?1 , status = ?2 , compression = ?3 , max_connections = ?4 , max_replication_lag = ?5 , use_ssl = ?6 , max_latency_ms = ?7 , comment = ?8 , gtid_port = ?9 WHERE hostgroup_id = ?10 AND hostname = ?11 AND port = ?12"; - //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query2, -1, &statement2, 0); - rc = mydb->prepare_v2(query2, &statement2); - ASSERT_SQLITE_OK(rc, mydb); - - for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { - SQLite3_row* r = *it; - long long ptr = atoll(r->fields[12]); // increase this index every time a new column is added - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), ptr, atoi(r->fields[0]), atoi(r->fields[6])); - //fprintf(stderr,"%lld\n", ptr); - if (ptr == 0) { - if (GloMTH->variables.hostgroup_manager_verbose) { - proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5])); - } - MySrvC* mysrvc = new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), atoi(r->fields[10]), r->fields[11]); // add new fields here if adding more columns in mysql_servers - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), mysrvc, atoi(r->fields[0])); - MyHGM->add(mysrvc, atoi(r->fields[0])); - ptr = (uintptr_t)mysrvc; - rc = (*proxy_sqlite3_bind_int64)(statement1, 1, ptr); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_text)(statement1, 3, r->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, mydb); - SAFE_SQLITE3_STEP2(statement1); - rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); - if (mysrvc->gtid_port) { - // this server has gtid_port configured, we set use_gtid - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); - use_gtid = true; - } - } else { - bool run_update = false; - MySrvC* mysrvc = (MySrvC*)ptr; - // carefully increase the 2nd index by 1 for every new column added - if (atoi(r->fields[3]) != atoi(r->fields[13])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing gtid_port for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), mysrvc->gtid_port, atoi(r->fields[13])); - mysrvc->gtid_port = atoi(r->fields[13]); - } - - if (atoi(r->fields[4]) != atoi(r->fields[14])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), mysrvc->weight, atoi(r->fields[14])); - mysrvc->weight = atoi(r->fields[14]); - } - if (atoi(r->fields[5]) != atoi(r->fields[15])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing status for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[5]), mysrvc->status, atoi(r->fields[15])); - mysrvc->status = (MySerStatus)atoi(r->fields[15]); - if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED) { - mysrvc->shunned_automatic = false; - } - } - if (atoi(r->fields[6]) != atoi(r->fields[16])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing compression for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[6]), mysrvc->compression, atoi(r->fields[16])); - mysrvc->compression = atoi(r->fields[16]); - } - if (atoi(r->fields[7]) != atoi(r->fields[17])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing max_connections for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[7]), mysrvc->max_connections, atoi(r->fields[17])); - mysrvc->max_connections = atoi(r->fields[17]); - } - if (atoi(r->fields[8]) != atoi(r->fields[18])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing max_replication_lag for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[8]), mysrvc->max_replication_lag, atoi(r->fields[18])); - mysrvc->max_replication_lag = atoi(r->fields[18]); - if (mysrvc->max_replication_lag == 0) { // we just changed it to 0 - if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { - // the server is currently shunned due to replication lag - // but we reset max_replication_lag to 0 - // therefore we immediately reset the status too - mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; - } - } - } - if (atoi(r->fields[9]) != atoi(r->fields[19])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing use_ssl for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[9]), mysrvc->use_ssl, atoi(r->fields[19])); - mysrvc->use_ssl = atoi(r->fields[19]); - } - if (atoi(r->fields[10]) != atoi(r->fields[20])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing max_latency_ms for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[10]), mysrvc->max_latency_us / 1000, atoi(r->fields[20])); - mysrvc->max_latency_us = 1000 * atoi(r->fields[20]); - } - if (strcmp(r->fields[11], r->fields[21])) { - if (GloMTH->variables.hostgroup_manager_verbose) - proxy_info("Changing comment for server %d:%s:%d (%s:%d) from '%s' to '%s'\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[11], r->fields[21]); - free(mysrvc->comment); - mysrvc->comment = strdup(r->fields[21]); - } - if (run_update) { - rc = (*proxy_sqlite3_bind_int64)(statement2, 1, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 2, mysrvc->status); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 3, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 4, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 5, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 6, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 7, mysrvc->max_latency_us / 1000); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_text)(statement2, 8, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 9, mysrvc->gtid_port); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 10, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_text)(statement2, 11, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_bind_int64)(statement2, 12, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); - SAFE_SQLITE3_STEP2(statement2); - rc = (*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, mydb); - rc = (*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, mydb); - } - if (mysrvc->gtid_port) { - // this server has gtid_port configured, we set use_gtid - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); - use_gtid = true; - } - } - } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement2); - } - if (use_gtid) { - has_gtid_port = true; - } else { - has_gtid_port = false; - } - if (resultset) { delete resultset; resultset = NULL; } - - purge_mysql_servers_table(); - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); - mydb->execute("DELETE FROM mysql_servers"); - generate_mysql_servers_table(); - - const auto mysql_servers_checksum = get_mysql_servers_checksum(); - - update_hostgroup_manager_mappings(); - - char buf[80]; - uint32_t d32[2]; - memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); - sprintf(buf, "0x%0X%0X", d32[0], d32[1]); - pthread_mutex_lock(&GloVars.checksum_mutex); - GloVars.checksums_values.mysql_servers.set_checksum(buf); - GloVars.checksums_values.mysql_servers.version++; - //struct timespec ts; - //clock_gettime(CLOCK_REALTIME, &ts); - time_t t = time(NULL); - - if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && - GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { - GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; - } else { - GloVars.checksums_values.mysql_servers.epoch = t; - } - - GloVars.checksums_values.updates_cnt++; - GloVars.generate_global_checksum(); - GloVars.epoch_version = t; - pthread_mutex_unlock(&GloVars.checksum_mutex); - - update_table_mysql_servers_for_monitor(false); -} +///** +// * @brief Generates runtime mysql server records and checksum. +// * +// * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in +// * the specified order. +// * @param can be null or previously generated runtime_mysql_servers resultset can be passed. +// */ +//void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, +// const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server) { +// char* error = NULL; +// int cols = 0; +// int affected_rows = 0; +// SQLite3_result* resultset = NULL; +// // if any server has gtid_port enabled, use_gtid is set to true +// // and then has_gtid_port is set too +// bool use_gtid = false; +// +// mydb->execute("DELETE FROM mysql_servers"); +// generate_mysql_servers_table(); +// +// const char* query = "SELECT mem_pointer, t1.hostgroup_id, t1.hostname, t1.port FROM mysql_servers t1 LEFT OUTER JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE t2.hostgroup_id IS NULL"; +// mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); +// if (error) { +// proxy_error("Error on %s : %s\n", query, error); +// } else { +// if (GloMTH->variables.hostgroup_manager_verbose) { +// proxy_info("Dumping mysql_servers LEFT JOIN mysql_servers_incoming\n"); +// resultset->dump_to_stderr(); +// } +// +// for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { +// SQLite3_row* r = *it; +// long long ptr = atoll(r->fields[0]); +// proxy_warning("Removed server at address %lld, hostgroup %s, address %s port %s. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", ptr, r->fields[1], r->fields[2], r->fields[3]); +// MySrvC* mysrvc = (MySrvC*)ptr; +// mysrvc->status = MYSQL_SERVER_STATUS_OFFLINE_HARD; +// mysrvc->ConnectionsFree->drop_all_connections(); +// char* q1 = (char*)"DELETE FROM mysql_servers WHERE mem_pointer=%lld"; +// char* q2 = (char*)malloc(strlen(q1) + 32); +// sprintf(q2, q1, ptr); +// mydb->execute(q2); +// free(q2); +// } +// } +// if (resultset) { delete resultset; resultset = NULL; } +// +// mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); +// +// // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) +// query = (char*)"SELECT t1.*, t2.gtid_port, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.gtid_port<>t2.gtid_port OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; +// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); +// mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); +// if (error) { +// proxy_error("Error on %s : %s\n", query, error); +// } else { +// +// if (GloMTH->variables.hostgroup_manager_verbose) { +// proxy_info("Dumping mysql_servers JOIN mysql_servers_incoming\n"); +// resultset->dump_to_stderr(); +// } +// // optimization #829 +// int rc; +// sqlite3_stmt* statement1 = NULL; +// sqlite3_stmt* statement2 = NULL; +// //sqlite3 *mydb3=mydb->get_db(); +// char* query1 = (char*)"UPDATE mysql_servers SET mem_pointer = ?1 WHERE hostgroup_id = ?2 AND hostname = ?3 AND port = ?4"; +// //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); +// rc = mydb->prepare_v2(query1, &statement1); +// ASSERT_SQLITE_OK(rc, mydb); +// char* query2 = (char*)"UPDATE mysql_servers SET weight = ?1 , status = ?2 , compression = ?3 , max_connections = ?4 , max_replication_lag = ?5 , use_ssl = ?6 , max_latency_ms = ?7 , comment = ?8 , gtid_port = ?9 WHERE hostgroup_id = ?10 AND hostname = ?11 AND port = ?12"; +// //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query2, -1, &statement2, 0); +// rc = mydb->prepare_v2(query2, &statement2); +// ASSERT_SQLITE_OK(rc, mydb); +// +// for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { +// SQLite3_row* r = *it; +// long long ptr = atoll(r->fields[12]); // increase this index every time a new column is added +// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), ptr, atoi(r->fields[0]), atoi(r->fields[6])); +// //fprintf(stderr,"%lld\n", ptr); +// if (ptr == 0) { +// if (GloMTH->variables.hostgroup_manager_verbose) { +// proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5])); +// } +// MySrvC* mysrvc = new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), atoi(r->fields[10]), r->fields[11]); // add new fields here if adding more columns in mysql_servers +// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), mysrvc, atoi(r->fields[0])); +// MyHGM->add(mysrvc, atoi(r->fields[0])); +// ptr = (uintptr_t)mysrvc; +// rc = (*proxy_sqlite3_bind_int64)(statement1, 1, ptr); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_text)(statement1, 3, r->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, mydb); +// SAFE_SQLITE3_STEP2(statement1); +// rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); +// if (mysrvc->gtid_port) { +// // this server has gtid_port configured, we set use_gtid +// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); +// use_gtid = true; +// } +// } else { +// bool run_update = false; +// MySrvC* mysrvc = (MySrvC*)ptr; +// // carefully increase the 2nd index by 1 for every new column added +// if (atoi(r->fields[3]) != atoi(r->fields[13])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing gtid_port for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), mysrvc->gtid_port, atoi(r->fields[13])); +// mysrvc->gtid_port = atoi(r->fields[13]); +// } +// +// if (atoi(r->fields[4]) != atoi(r->fields[14])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), mysrvc->weight, atoi(r->fields[14])); +// mysrvc->weight = atoi(r->fields[14]); +// } +// if (atoi(r->fields[5]) != atoi(r->fields[15])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing status for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[5]), mysrvc->status, atoi(r->fields[15])); +// mysrvc->status = (MySerStatus)atoi(r->fields[15]); +// if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED) { +// mysrvc->shunned_automatic = false; +// } +// } +// if (atoi(r->fields[6]) != atoi(r->fields[16])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing compression for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[6]), mysrvc->compression, atoi(r->fields[16])); +// mysrvc->compression = atoi(r->fields[16]); +// } +// if (atoi(r->fields[7]) != atoi(r->fields[17])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing max_connections for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[7]), mysrvc->max_connections, atoi(r->fields[17])); +// mysrvc->max_connections = atoi(r->fields[17]); +// } +// if (atoi(r->fields[8]) != atoi(r->fields[18])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing max_replication_lag for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[8]), mysrvc->max_replication_lag, atoi(r->fields[18])); +// mysrvc->max_replication_lag = atoi(r->fields[18]); +// if (mysrvc->max_replication_lag == 0) { // we just changed it to 0 +// if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { +// // the server is currently shunned due to replication lag +// // but we reset max_replication_lag to 0 +// // therefore we immediately reset the status too +// mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; +// } +// } +// } +// if (atoi(r->fields[9]) != atoi(r->fields[19])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing use_ssl for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[9]), mysrvc->use_ssl, atoi(r->fields[19])); +// mysrvc->use_ssl = atoi(r->fields[19]); +// } +// if (atoi(r->fields[10]) != atoi(r->fields[20])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing max_latency_ms for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[10]), mysrvc->max_latency_us / 1000, atoi(r->fields[20])); +// mysrvc->max_latency_us = 1000 * atoi(r->fields[20]); +// } +// if (strcmp(r->fields[11], r->fields[21])) { +// if (GloMTH->variables.hostgroup_manager_verbose) +// proxy_info("Changing comment for server %d:%s:%d (%s:%d) from '%s' to '%s'\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[11], r->fields[21]); +// free(mysrvc->comment); +// mysrvc->comment = strdup(r->fields[21]); +// } +// if (run_update) { +// rc = (*proxy_sqlite3_bind_int64)(statement2, 1, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 2, mysrvc->status); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 3, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 4, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 5, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 6, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 7, mysrvc->max_latency_us / 1000); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_text)(statement2, 8, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 9, mysrvc->gtid_port); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 10, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_text)(statement2, 11, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_bind_int64)(statement2, 12, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); +// SAFE_SQLITE3_STEP2(statement2); +// rc = (*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, mydb); +// rc = (*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, mydb); +// } +// if (mysrvc->gtid_port) { +// // this server has gtid_port configured, we set use_gtid +// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); +// use_gtid = true; +// } +// } +// } +// (*proxy_sqlite3_finalize)(statement1); +// (*proxy_sqlite3_finalize)(statement2); +// } +// if (use_gtid) { +// has_gtid_port = true; +// } else { +// has_gtid_port = false; +// } +// if (resultset) { delete resultset; resultset = NULL; } +// +// purge_mysql_servers_table(); +// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); +// mydb->execute("DELETE FROM mysql_servers"); +// generate_mysql_servers_table(); +// +// const auto mysql_servers_checksum = get_mysql_servers_checksum(); +// +// char buf[80]; +// uint32_t d32[2]; +// memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); +// sprintf(buf, "0x%0X%0X", d32[0], d32[1]); +// pthread_mutex_lock(&GloVars.checksum_mutex); +// GloVars.checksums_values.mysql_servers.set_checksum(buf); +// GloVars.checksums_values.mysql_servers.version++; +// //struct timespec ts; +// //clock_gettime(CLOCK_REALTIME, &ts); +// time_t t = time(NULL); +// +// if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && +// GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { +// GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; +// } else { +// GloVars.checksums_values.mysql_servers.epoch = t; +// } +// +// GloVars.checksums_values.updates_cnt++; +// GloVars.generate_global_checksum(); +// GloVars.epoch_version = t; +// pthread_mutex_unlock(&GloVars.checksum_mutex); +// +// update_hostgroup_manager_mappings(); +// update_table_mysql_servers_for_monitor(false); +//} // we always assume that the calling thread has acquired a rdlock() int MySQL_HostGroups_Manager::servers_add(SQLite3_result *resultset) { @@ -1908,9 +1907,11 @@ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { } bool MySQL_HostGroups_Manager::commit( - SQLite3_result* runtime_mysql_servers, SQLite3_result* mysql_servers_incoming, - const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming + SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, + SQLite3_result* mysql_servers_incoming, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming, + bool only_commit_runtime_mysql_servers ) { + // if only_commit_runtime_mysql_servers is true, mysql_servers_incoming resultset will not be entertained and will cause memory leak. unsigned long long curtime1=monotonic_time(); wrlock(); @@ -1939,9 +1940,9 @@ bool MySQL_HostGroups_Manager::commit( } if (resultset) { delete resultset; resultset=NULL; } } - char *query=NULL; + char *query=NULL; query=(char *)"SELECT mem_pointer, t1.hostgroup_id, t1.hostname, t1.port FROM mysql_servers t1 LEFT OUTER JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE t2.hostgroup_id IS NULL"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); } else { @@ -1969,15 +1970,14 @@ bool MySQL_HostGroups_Manager::commit( //mydb->execute("DELETE FROM mysql_servers"); //generate_mysql_servers_table(); -// INSERT OR IGNORE INTO mysql_servers SELECT ... FROM mysql_servers_incoming -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status, compression, max_connections) SELECT hostgroup_id, hostname, port, weight, status, compression, max_connections FROM mysql_servers_incoming\n"); + // INSERT OR IGNORE INTO mysql_servers SELECT ... FROM mysql_servers_incoming + // proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status, compression, max_connections) SELECT hostgroup_id, hostname, port, weight, status, compression, max_connections FROM mysql_servers_incoming\n"); mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); - // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) query=(char *)"SELECT t1.*, t2.gtid_port, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.gtid_port<>t2.gtid_port OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); } else { @@ -2120,74 +2120,75 @@ bool MySQL_HostGroups_Manager::commit( has_gtid_port = false; } if (resultset) { delete resultset; resultset=NULL; } - //proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); - //mydb->execute("DELETE FROM mysql_servers_incoming"); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); + mydb->execute("DELETE FROM mysql_servers_incoming"); - // replication - if (incoming_replication_hostgroups) { // this IF is extremely important, otherwise replication hostgroups may disappear - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_replication_hostgroups\n"); - mydb->execute("DELETE FROM mysql_replication_hostgroups"); - generate_mysql_replication_hostgroups_table(); - } - // group replication - if (incoming_group_replication_hostgroups) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_group_replication_hostgroups\n"); - mydb->execute("DELETE FROM mysql_group_replication_hostgroups"); - generate_mysql_group_replication_hostgroups_table(); - } + uint64_t mysql_servers_incoming_checksum = 0; - // galera - if (incoming_galera_hostgroups) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_galera_hostgroups\n"); - mydb->execute("DELETE FROM mysql_galera_hostgroups"); - generate_mysql_galera_hostgroups_table(); - } + if (only_commit_runtime_mysql_servers == false) { + // replication + if (incoming_replication_hostgroups) { // this IF is extremely important, otherwise replication hostgroups may disappear + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_replication_hostgroups\n"); + mydb->execute("DELETE FROM mysql_replication_hostgroups"); + generate_mysql_replication_hostgroups_table(); + } - // AWS Aurora - if (incoming_aws_aurora_hostgroups) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_aurora_hostgroups\n"); - mydb->execute("DELETE FROM mysql_aws_aurora_hostgroups"); - generate_mysql_aws_aurora_hostgroups_table(); - } + // group replication + if (incoming_group_replication_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_group_replication_hostgroups\n"); + mydb->execute("DELETE FROM mysql_group_replication_hostgroups"); + generate_mysql_group_replication_hostgroups_table(); + } - // hostgroup attributes - if (incoming_hostgroup_attributes) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_hostgroup_attributes\n"); - mydb->execute("DELETE FROM mysql_hostgroup_attributes"); - generate_mysql_hostgroup_attributes_table(); - } + // galera + if (incoming_galera_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_galera_hostgroups\n"); + mydb->execute("DELETE FROM mysql_galera_hostgroups"); + generate_mysql_galera_hostgroups_table(); + } + // AWS Aurora + if (incoming_aws_aurora_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_aurora_hostgroups\n"); + mydb->execute("DELETE FROM mysql_aws_aurora_hostgroups"); + generate_mysql_aws_aurora_hostgroups_table(); + } - //if (GloAdmin && GloAdmin->checksum_variables.checksum_mysql_servers) - { - mydb->execute("DELETE FROM mysql_servers"); - generate_mysql_servers_table(); + // hostgroup attributes + if (incoming_hostgroup_attributes) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_hostgroup_attributes\n"); + mydb->execute("DELETE FROM mysql_hostgroup_attributes"); + generate_mysql_hostgroup_attributes_table(); + } - const auto mysql_servers_checksum = get_mysql_servers_checksum(runtime_mysql_servers); - const auto mysql_servers_incoming_checksum = get_mysql_servers_incoming_checksum(mysql_servers_incoming, false); + mysql_servers_incoming_checksum = get_mysql_servers_incoming_checksum(mysql_servers_incoming, false); + } - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); - mydb->execute("DELETE FROM mysql_servers_incoming"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); - char buf[80]; - uint32_t d32[2]; - const time_t t = time(NULL); + const auto mysql_servers_checksum = get_mysql_servers_checksum(runtime_mysql_servers); - memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); - sprintf(buf, "0x%0X%0X", d32[0], d32[1]); - pthread_mutex_lock(&GloVars.checksum_mutex); - GloVars.checksums_values.mysql_servers.set_checksum(buf); - GloVars.checksums_values.mysql_servers.version++; - //struct timespec ts; - //clock_gettime(CLOCK_REALTIME, &ts); - if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && - GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { - GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; - } else { - GloVars.checksums_values.mysql_servers.epoch = t; - } + char buf[80]; + uint32_t d32[2]; + const time_t t = time(NULL); + memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); + pthread_mutex_lock(&GloVars.checksum_mutex); + GloVars.checksums_values.mysql_servers.set_checksum(buf); + GloVars.checksums_values.mysql_servers.version++; + //struct timespec ts; + //clock_gettime(CLOCK_REALTIME, &ts); + if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && + GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { + GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; + } else { + GloVars.checksums_values.mysql_servers.epoch = t; + } + + if (only_commit_runtime_mysql_servers == false) { memcpy(&d32, &mysql_servers_incoming_checksum, sizeof(mysql_servers_incoming_checksum)); sprintf(buf, "0x%0X%0X", d32[0], d32[1]); GloVars.checksums_values.mysql_servers_incoming.set_checksum(buf); @@ -2199,13 +2200,13 @@ bool MySQL_HostGroups_Manager::commit( } else { GloVars.checksums_values.mysql_servers_incoming.epoch = t; } - - GloVars.checksums_values.updates_cnt++; - GloVars.generate_global_checksum(); - GloVars.epoch_version = t; - pthread_mutex_unlock(&GloVars.checksum_mutex); } + GloVars.checksums_values.updates_cnt++; + GloVars.generate_global_checksum(); + GloVars.epoch_version = t; + pthread_mutex_unlock(&GloVars.checksum_mutex); + // fill Hostgroup_Manager_Mapping with latest records update_hostgroup_manager_mappings(); @@ -2244,56 +2245,67 @@ bool MySQL_HostGroups_Manager::commit( uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers) { //Note: GloVars.checksum_mutex needs to be locked - char* error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - char* query = (char*)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; - mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + SQLite3_result* resultset = nullptr; if (runtime_mysql_servers == nullptr) { char* error = NULL; int cols = 0; int affected_rows = 0; - SQLite3_result* resultset = NULL; mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); - // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster - // nodes, or relevant for checksum computation. - const size_t init_row_count = resultset->rows_count; - size_t rm_rows_count = 0; - const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - rm_rows_count += 1; - return true; + if (resultset) { + if (resultset->rows_count) { + // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster + // nodes, or relevant for checksum computation. + const size_t init_row_count = resultset->rows_count; + size_t rm_rows_count = 0; + const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + rm_rows_count += 1; + return true; + } else { + return false; + } + }; + resultset->rows.erase( + std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), + resultset->rows.end() + ); + resultset->rows_count = init_row_count - rm_rows_count; + + save_runtime_mysql_servers(resultset); } else { - return false; + delete resultset; + resultset = nullptr; } - }; - resultset->rows.erase( - std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), - resultset->rows.end() - ); - resultset->rows_count = init_row_count - rm_rows_count; - - save_runtime_mysql_servers(resultset); + } else { + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); + } } else { + resultset = runtime_mysql_servers; save_runtime_mysql_servers(runtime_mysql_servers); } - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = 0; - - if (resultset) { - if (resultset->rows_count) { - uint64_t hash1_ = resultset->raw_checksum(); - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); - } - delete resultset; - } else { - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); - } + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = compute_mysql_servers_raw_checksum(resultset); + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]); + + //char* error = NULL; + //int cols = 0; + //int affected_rows = 0; + //SQLite3_result* resultset = NULL; + //char* query = (char*)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; + //mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + //if (resultset) { + // if (resultset->rows_count) { + // uint64_t hash1_ = resultset->raw_checksum(); + // table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; + // proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); + // } + // delete resultset; + //} else { + // proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); + //} return table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; } @@ -2302,14 +2314,14 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_r bool use_precalculated_checksum) { //Note: GloVars.checksum_mutex needs to be locked - - char* error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; + SQLite3_result* resultset = nullptr; if (incoming_mysql_servers == nullptr) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + mydb->execute_statement(MYHGM_GEN_INCOMING_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); if (resultset) { if (resultset->rows_count) { @@ -2331,80 +2343,31 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_r resultset->rows_count = init_row_count - rm_rows_count; save_mysql_servers_incoming(resultset); + } else { + delete resultset; + resultset = nullptr; } } else { proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", (long unsigned int)0); } } else { + resultset = incoming_mysql_servers; save_mysql_servers_incoming(incoming_mysql_servers); } if (use_precalculated_checksum == false) { - // reset all checksum - table_resultset_checksum.fill(0); + // reset checksum + //table_resultset_checksum[MYSQL_SERVERS_INCOMING] = 0; + table_resultset_checksum[MYSQL_REPLICATION_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_GROUP_REPLICATION_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_GALERA_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_AWS_AURORA_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_HOSTGROUP_ATTRIBUTES] = 0; } - resultset = get_current_mysql_table("mysql_servers_incoming"); - - if (resultset) { - int status_idx = -1; - - for (int i = 0; i < resultset->columns; i++) { - if (resultset->column_definition[i] && resultset->column_definition[i]->name && - strcasecmp(resultset->column_definition[i]->name, "status") == 0) { - status_idx = i; - break; - } - } - - if (resultset->rows_count) { - uint64_t hash1 = 0, hash2 = 0; - SpookyHash myhash; - myhash.Init(19, 3); - - for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { - SQLite3_row* r = *it; - - const char* status_conv = nullptr; - const char* status = r->fields[status_idx]; - - if (status) { - if (strcasecmp(status, "OFFLINE_HARD") == 0) - continue; - - if (strcasecmp(status, "ONLINE") == 0 || - strcasecmp(status, "SHUNNED") == 0) { - status_conv = "0"; - } else if (strcasecmp(status, "OFFLINE_SOFT") == 0) { - status_conv = "2"; - } - } - - for (int i = 0; i < resultset->columns; i++) { - - if (i != status_idx) { - if (r->fields[i]) { - myhash.Update(r->fields[i], r->sizes[i]); - } else { - myhash.Update("", 0); - } - } else { - if (status_conv) { - myhash.Update(status_conv, strlen(status_conv)); - } else { - myhash.Update("", 0); - } - } - } - } - myhash.Final(&hash1, &hash2); - - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_INCOMING] = hash1; - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", hash1); - } - } + table_resultset_checksum[MYSQL_SERVERS_INCOMING] = compute_mysql_servers_raw_checksum(resultset); + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", table_resultset_checksum[MYSQL_SERVERS_INCOMING]); - /*char* query = (char*)"SELECT hostgroup_id,hostname,port,gtid_port,CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, \ weight,compression,max_connections, max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_incoming WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); @@ -5154,11 +5117,7 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listexecute("DELETE FROM mysql_servers"); generate_mysql_servers_table(); - // reset hgsm_mysql_servers_checksum checksum - hgsm_mysql_servers_checksum = 0; - const auto mysql_servers_checksum = get_mysql_servers_checksum(); - hgsm_mysql_servers_checksum = mysql_servers_checksum; char buf[80]; @@ -8169,3 +8128,66 @@ void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::remove_HGM(MySrvC* srv) srv->status = MYSQL_SERVER_STATUS_OFFLINE_HARD; srv->ConnectionsFree->drop_all_connections(); } + +/** + * @brief This function computes the checksum of the mysql_servers resultset. +* As the checksum is being calculated, the function replaces the status values with their respective integer values. + * + * @param mysql_servers resultset of mysql_servers or mysql_servers_incoming. + */ +uint64_t MySQL_HostGroups_Manager::compute_mysql_servers_raw_checksum(const SQLite3_result* mysql_servers) { + + if (!mysql_servers || mysql_servers->rows_count == 0) + return 0; + + int status_idx = -1; + + for (int i = 0; i < mysql_servers->columns; i++) { + if (mysql_servers->column_definition[i] && mysql_servers->column_definition[i]->name && + strcmp(mysql_servers->column_definition[i]->name, "status") == 0) { + status_idx = i; + break; + } + } + + if (status_idx == -1) assert(0); + + SpookyHash myhash; + myhash.Init(19, 3); + + for (const SQLite3_row* r : mysql_servers->rows) { + + const char* mapped_status = ""; + const char* status = r->fields[status_idx]; + + if (status) { + if (strcasecmp(status, "OFFLINE_HARD") == 0) + continue; + + if (strcasecmp(status, "ONLINE") == 0 || + strcasecmp(status, "SHUNNED") == 0) { + mapped_status = "0"; + } else if (strcasecmp(status, "OFFLINE_SOFT") == 0) { + mapped_status = "2"; + } + } + + for (int i = 0; i < mysql_servers->columns; i++) { + + if (r->fields[i]) { + if (i != status_idx) { + myhash.Update(r->fields[i], r->sizes[i]); + } else { + myhash.Update(mapped_status, strlen(mapped_status)); + } + } else { + myhash.Update("", 0); + } + } + } + + uint64_t res_hash = 0, hash2 = 0; + myhash.Final(&res_hash, &hash2); + + return res_hash; +} \ No newline at end of file diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 0e53edd36c..5b351ff8ec 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -12059,7 +12059,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc } // commit all the changes - MyHGM->commit(runtime_mysql_servers, incoming_mysql_servers, peer_runtime_mysql_server, peer_mysql_server_incoming); + MyHGM->commit(runtime_mysql_servers, peer_runtime_mysql_server, incoming_mysql_servers, peer_mysql_server_incoming); // quering runtime table will update and return latest records, so this is not needed. // GloAdmin->save_mysql_servers_runtime_to_database(true); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 292e8f6faf..d47ebc287e 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1736,7 +1736,7 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading runtime_mysql_servers from peer %s:%d into mysql_servers_incoming", hostname, port); MyHGM->servers_add(runtime_mysql_servers_resultset.get()); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Updating runtime_mysql_servers from peer %s:%d", hostname, port); - MyHGM->update_runtime_mysql_servers_table(runtime_mysql_servers_resultset.get(), peer_runtime_mysql_server); + MyHGM->commit(runtime_mysql_servers_resultset.release(), peer_runtime_mysql_server, nullptr, {}, true); MyHGM->wrunlock(); // free result @@ -2847,7 +2847,7 @@ bool ProxySQL_Cluster_Nodes::Update_Global_Checksum(char * _h, uint16_t _p, MYSQ proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum 0x%llX for peer %s:%d matches\n", v, node->get_hostname(), node->get_port()); ret = false; } else { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum for peer %s:%d is different from fetched one. Local checksum:[0x%llX] Fetched checksum:[0x%llX]\n", node->get_hostname(), node->get_port(), node->global_checksum, v); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum for peer %s:%d is different from fetched one. Local checksum:[0x%lX] Fetched checksum:[0x%lX]\n", node->get_hostname(), node->get_port(), node->global_checksum, v); node->global_checksum = v; } } From 3821f562a895f1082efd7af86e48c9263996b094 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Fri, 7 Apr 2023 01:36:05 +0500 Subject: [PATCH 04/56] * Added comments * Few issues fixed --- lib/MySQL_HostGroups_Manager.cpp | 19 ++++++++++++++++--- lib/ProxySQL_Admin.cpp | 17 ++++++++++++++++- lib/ProxySQL_Cluster.cpp | 4 ++-- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 3fb6fa1da0..bdbdc85443 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1842,6 +1842,13 @@ void MySQL_HostGroups_Manager::commit_update_checksums_from_tables() { CUCFT1("mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); } +/** + * @brief This code updates the 'hostgroup_server_mapping' table with the most recent mysql_servers and mysql_replication_hostgroups + * records while utilizing checksums to prevent unnecessary updates. + * + * IMPORTANT: Make sure wrlock() is called before calling this method. + * +*/ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { if (hgsm_mysql_servers_checksum != table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] || @@ -1912,6 +1919,11 @@ bool MySQL_HostGroups_Manager::commit( bool only_commit_runtime_mysql_servers ) { // if only_commit_runtime_mysql_servers is true, mysql_servers_incoming resultset will not be entertained and will cause memory leak. + if (only_commit_runtime_mysql_servers) { + proxy_info("Generating runtime mysql servers records only"); + } else { + proxy_info("Generating runtime mysql servers and mysql servers incoming records"); + } unsigned long long curtime1=monotonic_time(); wrlock(); @@ -2120,9 +2132,6 @@ bool MySQL_HostGroups_Manager::commit( has_gtid_port = false; } if (resultset) { delete resultset; resultset=NULL; } - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); - mydb->execute("DELETE FROM mysql_servers_incoming"); - uint64_t mysql_servers_incoming_checksum = 0; @@ -2165,6 +2174,10 @@ bool MySQL_HostGroups_Manager::commit( mysql_servers_incoming_checksum = get_mysql_servers_incoming_checksum(mysql_servers_incoming, false); } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); + mydb->execute("DELETE FROM mysql_servers_incoming"); + + // regenerating mysql_servers records mydb->execute("DELETE FROM mysql_servers"); generate_mysql_servers_table(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 5b351ff8ec..c8aab8a15a 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -3705,7 +3705,22 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { GloAdmin->mysql_servers_wrunlock(); if (resultset == nullptr) { - resultset=MyHGM->dump_table_mysql(tn); + + // fetching mysql_servers records from admin table as HGM mysql_servers records might be different. + if (tn == "mysql_servers_incoming") { + char *error=NULL; + int cols=0; + int affected_rows=0; + const char* query = "SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + + GloAdmin->mysql_servers_wrlock(); + GloAdmin->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + GloAdmin->mysql_servers_wrunlock(); + } else { + resultset = MyHGM->dump_table_mysql(tn); + } + if (resultset) { sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); delete resultset; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index d47ebc287e..e3a5b9ba41 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1731,13 +1731,13 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); if (computed_checksum == peer_checksum) { - MyHGM->wrlock(); + GloAdmin->mysql_servers_wrlock(); std::unique_ptr runtime_mysql_servers_resultset = get_SQLite3_resulset(result); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading runtime_mysql_servers from peer %s:%d into mysql_servers_incoming", hostname, port); MyHGM->servers_add(runtime_mysql_servers_resultset.get()); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Updating runtime_mysql_servers from peer %s:%d", hostname, port); MyHGM->commit(runtime_mysql_servers_resultset.release(), peer_runtime_mysql_server, nullptr, {}, true); - MyHGM->wrunlock(); + GloAdmin->mysql_servers_wrunlock(); // free result mysql_free_result(result); From 11229888b4e80542567d956c759d5691528c597f Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Sat, 8 Apr 2023 01:52:33 +0500 Subject: [PATCH 05/56] * Rename mysql_servers_incoming to mysql_servers_v2 * Fetching mysql_servers_v2 records from admin * Added comments. --- include/MySQL_HostGroups_Manager.h | 15 +-- include/ProxySQL_Cluster.hpp | 18 +-- include/proxysql_admin.h | 12 +- include/proxysql_glovars.hpp | 2 +- lib/MySQL_HostGroups_Manager.cpp | 127 ++++++++++----------- lib/ProxySQL_Admin.cpp | 62 +++++++---- lib/ProxySQL_Cluster.cpp | 170 ++++++++++++++--------------- lib/ProxySQL_GloVars.cpp | 2 +- 8 files changed, 203 insertions(+), 205 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index ee313df312..7295e6b75d 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -66,7 +66,8 @@ using json = nlohmann::json; #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" -#define MYHGM_GEN_INCOMING_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming ORDER BY hostgroup_id, hostname, port" +//#define MYHGM_GEN_INCOMING_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming ORDER BY hostgroup_id, hostname, port" +#define MYHGM_GEN_ADMIN_MYSQL_SERVERS "SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port" typedef std::unordered_map umap_mysql_errors; @@ -393,7 +394,7 @@ class MySQL_HostGroups_Manager { #endif enum HGM_TABLES { - MYSQL_SERVERS_INCOMING = 0, + MYSQL_SERVERS_V2 = 0, MYSQL_REPLICATION_HOSTGROUPS, MYSQL_GROUP_REPLICATION_HOSTGROUPS, MYSQL_GALERA_HOSTGROUPS, @@ -553,7 +554,7 @@ class MySQL_HostGroups_Manager { void generate_mysql_hostgroup_attributes_table(); SQLite3_result *incoming_hostgroup_attributes; - SQLite3_result* incoming_mysql_servers; + SQLite3_result* incoming_mysql_servers_v2; std::thread *HGCU_thread; @@ -700,7 +701,7 @@ class MySQL_HostGroups_Manager { int servers_add(SQLite3_result *resultset); //void update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, - SQLite3_result* mysql_servers_incoming = nullptr, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming = {}, + SQLite3_result* mysql_servers_v2 = nullptr, const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {}, bool only_commit_runtime_mysql_servers = false); void commit_update_checksums_from_tables(); void CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() @@ -713,11 +714,11 @@ class MySQL_HostGroups_Manager { void save_runtime_mysql_servers(SQLite3_result *); /** - * @brief Store the resultset for the 'mysql_servers_incoming' table. + * @brief Store the resultset for the 'mysql_servers_v2' table. * The store configuration is later used by Cluster to propagate current config. * @param The resulset to be stored replacing the current one. */ - void save_mysql_servers_incoming(SQLite3_result* s); + void save_mysql_servers_v2(SQLite3_result* s); /** * @brief These setters/getter functions store and retrieve the currently hold resultset for the @@ -831,7 +832,7 @@ class MySQL_HostGroups_Manager { void update_hostgroup_manager_mappings(); uint64_t get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers = nullptr); - uint64_t get_mysql_servers_incoming_checksum(SQLite3_result* incoming_mysql_servers, bool use_precalculated_checksum = true); + uint64_t get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2 = nullptr); }; #endif /* __CLASS_MYSQL_HOSTGROUPS_MANAGER_H */ diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index b1dbf70199..71254e4b69 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -22,7 +22,7 @@ * - 'runtime_mysql_servers': tables 'mysql_servers' * - 'runtime_mysql_users'. * - 'runtime_mysql_query_rules'. - * - 'mysql_servers_incoming': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups', + * - 'mysql_servers_v2': tables admin 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups', * 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'. * IMPORTANT: For further clarify this means that it's important that the actual resultset produced by the intercepted * query preserve the filtering and ordering expressed in this queries. @@ -31,8 +31,8 @@ /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_servers'. See top comment for details. */ #define CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" -/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'mysql_servers_incoming'. See top comment for details. */ -#define CLUSTER_QUERY_MYSQL_SERVERS_INCOMING "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'mysql_servers_v2'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_SERVERS_V2 "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_v2 WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_replication_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup" @@ -193,7 +193,7 @@ class ProxySQL_Node_Entry { ProxySQL_Checksum_Value_2 mysql_servers; ProxySQL_Checksum_Value_2 mysql_users; ProxySQL_Checksum_Value_2 proxysql_servers; - ProxySQL_Checksum_Value_2 mysql_servers_incoming; + ProxySQL_Checksum_Value_2 mysql_servers_v2; } checksums_values; uint64_t global_checksum; }; @@ -285,7 +285,7 @@ class ProxySQL_Cluster_Nodes { SQLite3_result * stats_proxysql_servers_metrics(); void get_peer_to_sync_mysql_query_rules(char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_runtime_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address); - void get_peer_to_sync_mysql_servers_incoming(char** host, uint16_t* port, char** peer_mysql_servers_incoming_checksum, + void get_peer_to_sync_mysql_servers_v2(char** host, uint16_t* port, char** peer_mysql_servers_v2_checksum, char** peer_runtime_mysql_servers_checksum, char** ip_address); void get_peer_to_sync_mysql_users(char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_mysql_variables(char **host, uint16_t *port, char** ip_address); @@ -374,8 +374,8 @@ struct variable_type { }; enum class mysql_servers_sync_algorithm { - runtime_mysql_servers_and_mysql_servers_incoming = 1, // sync runtime_mysql_servers and mysql_server_incoming from remote peer - mysql_servers_incoming = 2, // sync mysql_server_incoming from remote peer + runtime_mysql_servers_and_mysql_servers_v2 = 1, // sync runtime_mysql_servers and mysql_server_v2 from remote peer + mysql_servers_v2 = 2, // sync mysql_server_v2 from remote peer auto_select = 3 // based on -M flag }; @@ -407,7 +407,7 @@ class ProxySQL_Cluster { public: pthread_mutex_t update_mysql_query_rules_mutex; pthread_mutex_t update_runtime_mysql_servers_mutex; - pthread_mutex_t update_mysql_servers_incoming_mutex; + pthread_mutex_t update_mysql_servers_v2_mutex; pthread_mutex_t update_mysql_users_mutex; pthread_mutex_t update_mysql_variables_mutex; pthread_mutex_t update_proxysql_servers_mutex; @@ -486,7 +486,7 @@ class ProxySQL_Cluster { void join_term_thread(); void pull_mysql_query_rules_from_peer(const std::string& expected_checksum, const time_t epoch); void pull_runtime_mysql_servers_from_peer(const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); - void pull_mysql_servers_incoming_from_peer(const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming, + void pull_mysql_servers_v2_from_peer(const mysql_servers_v2_checksum_t& peer_mysql_server_v2, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, bool fetch_runtime_mysql_servers = false); void pull_mysql_users_from_peer(const std::string& expected_checksum, const time_t epoch); /** diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index ca3a3e4e04..f55c8a64e6 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -117,7 +117,7 @@ struct admin_metrics_map_idx { extern int admin__web_verbosity; struct incoming_servers_t { - SQLite3_result* incoming_mysql_servers = NULL; + SQLite3_result* incoming_mysql_servers_v2 = NULL; SQLite3_result* runtime_mysql_servers = NULL; SQLite3_result* incoming_replication_hostgroups = NULL; SQLite3_result* incoming_group_replication_hostgroups = NULL; @@ -129,7 +129,7 @@ struct incoming_servers_t { incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); }; -// Seperate structs for runtime mysql server and mysql server incoming to avoid human error +// Seperate structs for runtime mysql server and mysql server v2 to avoid human error struct runtime_mysql_servers_checksum_t { std::string checksum; time_t epoch; @@ -138,12 +138,12 @@ struct runtime_mysql_servers_checksum_t { runtime_mysql_servers_checksum_t(const std::string& checksum, time_t epoch); }; -struct mysql_servers_incoming_checksum_t { +struct mysql_servers_v2_checksum_t { std::string checksum; time_t epoch; - mysql_servers_incoming_checksum_t(); - mysql_servers_incoming_checksum_t(const std::string& checksum, time_t epoch); + mysql_servers_v2_checksum_t(); + mysql_servers_v2_checksum_t(const std::string& checksum, time_t epoch); }; // @@ -419,7 +419,7 @@ class ProxySQL_Admin { void flush_admin_variables__from_memory_to_disk(); void flush_ldap_variables__from_memory_to_disk(); void load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers = {}, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, - const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming = {}); + const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {}); void save_mysql_servers_from_runtime(); /** * @brief Performs the load to runtime of the current configuration in 'main' for 'mysql_query_rules' and diff --git a/include/proxysql_glovars.hpp b/include/proxysql_glovars.hpp index 66c9424e0e..395d1f1e41 100644 --- a/include/proxysql_glovars.hpp +++ b/include/proxysql_glovars.hpp @@ -133,7 +133,7 @@ class ProxySQL_GlobalVariables { ProxySQL_Checksum_Value mysql_variables; ProxySQL_Checksum_Value ldap_variables; ProxySQL_Checksum_Value proxysql_servers; - ProxySQL_Checksum_Value mysql_servers_incoming; + ProxySQL_Checksum_Value mysql_servers_v2; uint64_t global_checksum; unsigned long long updates_cnt; unsigned long long dumped_at; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index bdbdc85443..ac2319cd6e 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1356,7 +1356,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { incoming_galera_hostgroups=NULL; incoming_aws_aurora_hostgroups = NULL; incoming_hostgroup_attributes = NULL; - incoming_mysql_servers = NULL; + incoming_mysql_servers_v2 = NULL; pthread_rwlock_init(>id_rwlock, NULL); gtid_missing_nodes = false; gtid_ev_loop=NULL; @@ -1915,14 +1915,14 @@ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { bool MySQL_HostGroups_Manager::commit( SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, - SQLite3_result* mysql_servers_incoming, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming, + SQLite3_result* mysql_servers_v2, const mysql_servers_v2_checksum_t& peer_mysql_server_v2, bool only_commit_runtime_mysql_servers ) { - // if only_commit_runtime_mysql_servers is true, mysql_servers_incoming resultset will not be entertained and will cause memory leak. + // if only_commit_runtime_mysql_servers is true, mysql_servers_v2 resultset will not be entertained and will cause memory leak. if (only_commit_runtime_mysql_servers) { - proxy_info("Generating runtime mysql servers records only"); + proxy_info("Generating runtime mysql servers records only.\n"); } else { - proxy_info("Generating runtime mysql servers and mysql servers incoming records"); + proxy_info("Generating runtime mysql servers and mysql servers v2 records.\n"); } unsigned long long curtime1=monotonic_time(); @@ -2133,7 +2133,7 @@ bool MySQL_HostGroups_Manager::commit( } if (resultset) { delete resultset; resultset=NULL; } - uint64_t mysql_servers_incoming_checksum = 0; + uint64_t mysql_servers_v2_checksum = 0; if (only_commit_runtime_mysql_servers == false) { // replication @@ -2171,7 +2171,7 @@ bool MySQL_HostGroups_Manager::commit( generate_mysql_hostgroup_attributes_table(); } - mysql_servers_incoming_checksum = get_mysql_servers_incoming_checksum(mysql_servers_incoming, false); + mysql_servers_v2_checksum = get_mysql_servers_v2_checksum(mysql_servers_v2); } proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); @@ -2202,16 +2202,16 @@ bool MySQL_HostGroups_Manager::commit( } if (only_commit_runtime_mysql_servers == false) { - memcpy(&d32, &mysql_servers_incoming_checksum, sizeof(mysql_servers_incoming_checksum)); + memcpy(&d32, &mysql_servers_v2_checksum, sizeof(mysql_servers_v2_checksum)); sprintf(buf, "0x%0X%0X", d32[0], d32[1]); - GloVars.checksums_values.mysql_servers_incoming.set_checksum(buf); - GloVars.checksums_values.mysql_servers_incoming.version++; + GloVars.checksums_values.mysql_servers_v2.set_checksum(buf); + GloVars.checksums_values.mysql_servers_v2.version++; - if (peer_mysql_server_incoming.epoch != 0 && peer_mysql_server_incoming.checksum.empty() == false && - GloVars.checksums_values.mysql_servers_incoming.checksum == peer_mysql_server_incoming.checksum) { - GloVars.checksums_values.mysql_servers_incoming.epoch = peer_mysql_server_incoming.epoch; + if (peer_mysql_server_v2.epoch != 0 && peer_mysql_server_v2.checksum.empty() == false && + GloVars.checksums_values.mysql_servers_v2.checksum == peer_mysql_server_v2.checksum) { + GloVars.checksums_values.mysql_servers_v2.epoch = peer_mysql_server_v2.epoch; } else { - GloVars.checksums_values.mysql_servers_incoming.epoch = t; + GloVars.checksums_values.mysql_servers_v2.epoch = t; } } @@ -2255,6 +2255,16 @@ bool MySQL_HostGroups_Manager::commit( return true; } +/** + * @brief Calculate the checksum for the runtime mysql_servers record, after excluding all the rows + * with the status OFFLINE_HARD from the result set + * + * @details The runtime mysql_servers is now considered as a distinct module and have a separate checksum calculation. + * This is because the records in the runtime module may differ from those in the admin mysql_servers module, which + * can cause synchronization issues within the cluster. + * + * @param runtime_mysql_servers resultset of runtime mysql_servers or can be a nullptr. +*/ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers) { //Note: GloVars.checksum_mutex needs to be locked @@ -2302,40 +2312,29 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = compute_mysql_servers_raw_checksum(resultset); proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]); - - //char* error = NULL; - //int cols = 0; - //int affected_rows = 0; - //SQLite3_result* resultset = NULL; - //char* query = (char*)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; - //mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); - //if (resultset) { - // if (resultset->rows_count) { - // uint64_t hash1_ = resultset->raw_checksum(); - // table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; - // proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); - // } - // delete resultset; - //} else { - // proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); - //} return table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; } -uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_result* incoming_mysql_servers, - bool use_precalculated_checksum) { +/** + * @brief Computes checksum for the admin mysql_server resultset and generates an accumulated checksum by including the following modules: + * MYSQL_REPLICATION_HOSTGROUPS, MYSQL_GROUP_REPLICATION_HOSTGROUPS, MYSQL_GALERA_HOSTGROUPS, and MYSQL_HOSTGROUP_ATTRIBUTES. + * + * @param incoming_mysql_servers_v2 resultset of admin mysql_servers or can be a nullptr. + */ +uint64_t MySQL_HostGroups_Manager::get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2) { //Note: GloVars.checksum_mutex needs to be locked SQLite3_result* resultset = nullptr; - if (incoming_mysql_servers == nullptr) + if (incoming_mysql_servers_v2 == nullptr) { char* error = nullptr; int cols = 0; int affected_rows = 0; - mydb->execute_statement(MYHGM_GEN_INCOMING_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); + GloAdmin->admindb->execute_statement(MYHGM_GEN_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); + if (resultset) { if (resultset->rows_count) { @@ -2355,52 +2354,36 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_incoming_checksum(SQLite3_r ); resultset->rows_count = init_row_count - rm_rows_count; - save_mysql_servers_incoming(resultset); + save_mysql_servers_v2(resultset); } else { delete resultset; resultset = nullptr; } } else { - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", (long unsigned int)0); + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_v2", (long unsigned int)0); } } else { - resultset = incoming_mysql_servers; - save_mysql_servers_incoming(incoming_mysql_servers); + resultset = incoming_mysql_servers_v2; + save_mysql_servers_v2(incoming_mysql_servers_v2); } - if (use_precalculated_checksum == false) { - // reset checksum - //table_resultset_checksum[MYSQL_SERVERS_INCOMING] = 0; - table_resultset_checksum[MYSQL_REPLICATION_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_GROUP_REPLICATION_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_GALERA_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_AWS_AURORA_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_HOSTGROUP_ATTRIBUTES] = 0; - } + // reset checksum excluding MYSQL_SERVERS_V2 + table_resultset_checksum[MYSQL_REPLICATION_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_GROUP_REPLICATION_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_GALERA_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_AWS_AURORA_HOSTGROUPS] = 0; + table_resultset_checksum[MYSQL_HOSTGROUP_ATTRIBUTES] = 0; - table_resultset_checksum[MYSQL_SERVERS_INCOMING] = compute_mysql_servers_raw_checksum(resultset); - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", table_resultset_checksum[MYSQL_SERVERS_INCOMING]); - - /*char* query = (char*)"SELECT hostgroup_id,hostname,port,gtid_port,CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, \ - weight,compression,max_connections, max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_incoming WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; - mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); - if (resultset) { - if (resultset->rows_count) { - uint64_t hash1_ = resultset->raw_checksum(); - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_INCOMING] = hash1_; - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_incoming", hash1_); - } - }*/ + table_resultset_checksum[MYSQL_SERVERS_V2] = compute_mysql_servers_raw_checksum(resultset); + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_v2", table_resultset_checksum[MYSQL_SERVERS_V2]); - if (use_precalculated_checksum == false) { - commit_update_checksums_from_tables(); - } + commit_update_checksums_from_tables(); uint64_t hash1 = 0, hash2 = 0; SpookyHash myhash; bool init = false; - hash1 = table_resultset_checksum[MYSQL_SERVERS_INCOMING]; + hash1 = table_resultset_checksum[MYSQL_SERVERS_V2]; if (hash1) { if (init == false) { init = true; @@ -4240,12 +4223,12 @@ void MySQL_HostGroups_Manager::save_runtime_mysql_servers(SQLite3_result *s) { runtime_mysql_servers=s; } -void MySQL_HostGroups_Manager::save_mysql_servers_incoming(SQLite3_result* s) { - if (incoming_mysql_servers) { - delete incoming_mysql_servers; - incoming_mysql_servers = nullptr; +void MySQL_HostGroups_Manager::save_mysql_servers_v2(SQLite3_result* s) { + if (incoming_mysql_servers_v2) { + delete incoming_mysql_servers_v2; + incoming_mysql_servers_v2 = nullptr; } - incoming_mysql_servers = s; + incoming_mysql_servers_v2 = s; } SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& name) { @@ -4261,8 +4244,8 @@ SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& return this->incoming_hostgroup_attributes; } else if (name == "mysql_servers") { return this->runtime_mysql_servers; - } else if (name == "mysql_servers_incoming") { - return this->incoming_mysql_servers; + } else if (name == "mysql_servers_v2") { + return this->incoming_mysql_servers_v2; } else { assert(0); } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index c8aab8a15a..d09511671d 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1058,7 +1058,7 @@ bool FlushCommandWrapper(MySQL_Session *sess, const string& modname, char *query incoming_servers_t::incoming_servers_t() {} incoming_servers_t::incoming_servers_t( - SQLite3_result* incoming_mysql_servers, + SQLite3_result* incoming_mysql_servers_v2, SQLite3_result* incoming_replication_hostgroups, SQLite3_result* incoming_group_replication_hostgroups, SQLite3_result* incoming_galera_hostgroups, @@ -1066,7 +1066,7 @@ incoming_servers_t::incoming_servers_t( SQLite3_result* incoming_hostgroup_attributes, SQLite3_result* runtime_mysql_servers ) : - incoming_mysql_servers(incoming_mysql_servers), + incoming_mysql_servers_v2(incoming_mysql_servers_v2), incoming_replication_hostgroups(incoming_replication_hostgroups), incoming_group_replication_hostgroups(incoming_group_replication_hostgroups), incoming_galera_hostgroups(incoming_galera_hostgroups), @@ -1080,9 +1080,9 @@ runtime_mysql_servers_checksum_t::runtime_mysql_servers_checksum_t() : epoch(0) runtime_mysql_servers_checksum_t::runtime_mysql_servers_checksum_t(const std::string& checksum, time_t epoch) : checksum(checksum), epoch(epoch) {} -mysql_servers_incoming_checksum_t::mysql_servers_incoming_checksum_t() : epoch(0) {} +mysql_servers_v2_checksum_t::mysql_servers_v2_checksum_t() : epoch(0) {} -mysql_servers_incoming_checksum_t::mysql_servers_incoming_checksum_t(const std::string& checksum, time_t epoch) : +mysql_servers_v2_checksum_t::mysql_servers_v2_checksum_t(const std::string& checksum, time_t epoch) : checksum(checksum), epoch(epoch) {} @@ -3696,8 +3696,8 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { tn = "mysql_aws_aurora_hostgroups"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES, query_no_space, strlen(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES))) { tn = "mysql_hostgroup_attributes"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_INCOMING, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_INCOMING))) { - tn = "mysql_servers_incoming"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_V2, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_V2))) { + tn = "mysql_servers_v2"; } if (tn != "") { GloAdmin->mysql_servers_wrlock(); @@ -3706,17 +3706,36 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (resultset == nullptr) { - // fetching mysql_servers records from admin table as HGM mysql_servers records might be different. - if (tn == "mysql_servers_incoming") { + // This section of the code contains specific instructions for mysql_servers_v2, which is a virtual table + // that represents the mysql_servers (admin) records. In order to generate the resultset, data will be retrieved + // from mysql_server (admin) instead. + if (tn == "mysql_servers_v2") { char *error=NULL; int cols=0; int affected_rows=0; const char* query = "SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - GloAdmin->mysql_servers_wrlock(); GloAdmin->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); GloAdmin->mysql_servers_wrunlock(); + if (resultset && resultset->rows_count) { + const size_t init_row_count = resultset->rows_count; + size_t rm_rows_count = 0; + const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + rm_rows_count += 1; + return true; + } else { + return false; + } + }; + resultset->rows.erase( + std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), + resultset->rows.end() + ); + + resultset->rows_count = init_row_count - rm_rows_count; + } } else { resultset = MyHGM->dump_table_mysql(tn); } @@ -11072,10 +11091,10 @@ void ProxySQL_Admin::dump_checksums_values_table() { rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); - rc = (*proxy_sqlite3_bind_text)(statement1, 1, "mysql_servers_incoming", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.mysql_servers_incoming.version); ASSERT_SQLITE_OK(rc, admindb); - rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.mysql_servers_incoming.epoch); ASSERT_SQLITE_OK(rc, admindb); - rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.mysql_servers_incoming.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 1, "mysql_servers_v2", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.mysql_servers_v2.version); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.mysql_servers_v2.epoch); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.mysql_servers_v2.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); SAFE_SQLITE3_STEP2(statement1); rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); @@ -11881,7 +11900,7 @@ void ProxySQL_Admin::load_scheduler_to_runtime() { } void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers, - const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming) { + const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, const mysql_servers_v2_checksum_t& peer_mysql_server_v2) { // make sure that the caller has called mysql_servers_wrlock() char *error=NULL; int cols=0; @@ -11893,8 +11912,6 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; SQLite3_result *resultset_hostgroup_attributes=NULL; - SQLite3_result *resultset_mysql_servers_incoming = NULL; - SQLite3_result* resultset_mysql_servers_admin = NULL; SQLite3_result* runtime_mysql_servers = incoming_servers.runtime_mysql_servers; SQLite3_result* incoming_replication_hostgroups = incoming_servers.incoming_replication_hostgroups; @@ -11902,15 +11919,12 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc SQLite3_result* incoming_galera_hostgroups = incoming_servers.incoming_galera_hostgroups; SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups; SQLite3_result* incoming_hostgroup_attributes = incoming_servers.incoming_hostgroup_attributes; - SQLite3_result* incoming_mysql_servers = incoming_servers.incoming_mysql_servers; - - char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset_mysql_servers_admin); + SQLite3_result* incoming_mysql_servers_v2 = incoming_servers.incoming_mysql_servers_v2; + const char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; if (runtime_mysql_servers == nullptr) { - resultset_servers = resultset_mysql_servers_admin; - resultset_mysql_servers_admin = nullptr; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset_servers); } else { resultset_servers = runtime_mysql_servers; } @@ -12074,7 +12088,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc } // commit all the changes - MyHGM->commit(runtime_mysql_servers, peer_runtime_mysql_server, incoming_mysql_servers, peer_mysql_server_incoming); + MyHGM->commit(runtime_mysql_servers, peer_runtime_mysql_server, incoming_mysql_servers_v2, peer_mysql_server_v2); // quering runtime table will update and return latest records, so this is not needed. // GloAdmin->save_mysql_servers_runtime_to_database(true); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index e3a5b9ba41..1fddef4749 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -507,28 +507,28 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } continue; } - if (strcmp(row[0], "mysql_servers_incoming")==0) { - checksums_values.mysql_servers_incoming.version = atoll(row[1]); - checksums_values.mysql_servers_incoming.epoch = atoll(row[2]); - checksums_values.mysql_servers_incoming.last_updated = now; - if (strcmp(checksums_values.mysql_servers_incoming.checksum, row[3])) { - strcpy(checksums_values.mysql_servers_incoming.checksum, row[3]); - checksums_values.mysql_servers_incoming.last_changed = now; - checksums_values.mysql_servers_incoming.diff_check = 1; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for mysql_servers_incoming from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers_incoming.version, checksums_values.mysql_servers_incoming.epoch, checksums_values.mysql_servers_incoming.checksum); - proxy_info("Cluster: detected a new checksum for mysql_servers_incoming from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers_incoming.version, checksums_values.mysql_servers_incoming.epoch, checksums_values.mysql_servers_incoming.checksum); - if (strcmp(checksums_values.mysql_servers_incoming.checksum, GloVars.checksums_values.mysql_servers_incoming.checksum) == 0) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); - proxy_info("Cluster: checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); + if (strcmp(row[0], "mysql_servers_v2")==0) { + checksums_values.mysql_servers_v2.version = atoll(row[1]); + checksums_values.mysql_servers_v2.epoch = atoll(row[2]); + checksums_values.mysql_servers_v2.last_updated = now; + if (strcmp(checksums_values.mysql_servers_v2.checksum, row[3])) { + strcpy(checksums_values.mysql_servers_v2.checksum, row[3]); + checksums_values.mysql_servers_v2.last_changed = now; + checksums_values.mysql_servers_v2.diff_check = 1; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected new checksum for mysql_servers_v2 from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers_v2.version, checksums_values.mysql_servers_v2.epoch, checksums_values.mysql_servers_v2.checksum); + proxy_info("Cluster: detected a new checksum for mysql_servers_v2 from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_servers_v2.version, checksums_values.mysql_servers_v2.epoch, checksums_values.mysql_servers_v2.checksum); + if (strcmp(checksums_values.mysql_servers_v2.checksum, GloVars.checksums_values.mysql_servers_v2.checksum) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_v2 from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_v2.checksum); + proxy_info("Cluster: checksum for mysql_servers_v2 from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_servers_v2.checksum); } } else { - checksums_values.mysql_servers_incoming.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_servers_incoming.version, checksums_values.mysql_servers_incoming.epoch, - checksums_values.mysql_servers_incoming.checksum, GloVars.checksums_values.mysql_servers_incoming.checksum, checksums_values.mysql_servers_incoming.diff_check); + checksums_values.mysql_servers_v2.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_v2 from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_servers_v2.version, checksums_values.mysql_servers_v2.epoch, + checksums_values.mysql_servers_v2.checksum, GloVars.checksums_values.mysql_servers_v2.checksum, checksums_values.mysql_servers_v2.diff_check); } - if (strcmp(checksums_values.mysql_servers_incoming.checksum, GloVars.checksums_values.mysql_servers_incoming.checksum) == 0) { - checksums_values.mysql_servers_incoming.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_incoming from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_servers_incoming.checksum); + if (strcmp(checksums_values.mysql_servers_v2.checksum, GloVars.checksums_values.mysql_servers_v2.checksum) == 0) { + checksums_values.mysql_servers_v2.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_v2 from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_servers_v2.checksum); } continue; } @@ -656,9 +656,9 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } if (v->diff_check) v->diff_check++; - v = &checksums_values.mysql_servers_incoming; + v = &checksums_values.mysql_servers_v2; v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.mysql_servers_incoming.checksum) == 0) { + if (strcmp(v->checksum, GloVars.checksums_values.mysql_servers_v2.checksum) == 0) { v->diff_check = 0; } if (v->diff_check) @@ -741,13 +741,13 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (mysql_server_sync_algo == mysql_servers_sync_algorithm::auto_select) { mysql_server_sync_algo = (GloVars.global.monitor == false) ? - mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_incoming : mysql_servers_sync_algorithm::mysql_servers_incoming; + mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_v2 : mysql_servers_sync_algorithm::mysql_servers_v2; } - v = &checksums_values.mysql_servers_incoming; - const unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_incoming.version, 0); - const unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_incoming.epoch, 0); - const char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_incoming.checksum, 0); + v = &checksums_values.mysql_servers_v2; + const unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_v2.version, 0); + const unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_v2.epoch, 0); + const char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers_v2.checksum, 0); bool runtime_mysql_servers_already_loaded = false; if (v->version > 1) { @@ -757,28 +757,28 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (v->epoch > own_epoch) // epoch is newer ) { if (v->diff_check >= diff_ms) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - proxy_info("Cluster: detected a peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - if (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_incoming) { + if (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_v2) { ProxySQL_Checksum_Value_2* runtime_mysql_server_checksum = &checksums_values.mysql_servers; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_incoming and mysql_servers from peer %s:%d", hostname, port); - // fetching mysql server incoming and runtime mysql server - GloProxyCluster->pull_mysql_servers_incoming_from_peer({ v->checksum, v->epoch }, + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_v2 and mysql_servers from peer %s:%d", hostname, port); + // fetching mysql server v2 and runtime mysql server + GloProxyCluster->pull_mysql_servers_v2_from_peer({ v->checksum, v->epoch }, { runtime_mysql_server_checksum->checksum, runtime_mysql_server_checksum->epoch }, true); runtime_mysql_servers_already_loaded = true; } else { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_incoming from peer %s:%d", hostname, port); - // fetching mysql server incoming - // to avoid runtime_mysql_servers generating new epoch, we are passing mysql_server_incoming checksum and epoch - GloProxyCluster->pull_mysql_servers_incoming_from_peer({ v->checksum, v->epoch }, { v->checksum, v->epoch }); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_v2 from peer %s:%d", hostname, port); + // fetching mysql server v2 + // to avoid runtime_mysql_servers generating new epoch, we are passing mysql_server_v2 checksum and epoch + GloProxyCluster->pull_mysql_servers_v2_from_peer({ v->checksum, v->epoch }, { v->checksum, v->epoch }); } } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms * 10)) == 0)) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); - proxy_error("Cluster: detected a peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); + proxy_error("Cluster: detected a peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms * 10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_servers_share_epoch]->Increment(); } } else { @@ -789,7 +789,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } } - if (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_incoming && + if (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_v2 && runtime_mysql_servers_already_loaded == false) { v = &checksums_values.mysql_servers; unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.version, 0); @@ -1775,23 +1775,23 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ } /** - * @brief mysql_servers_incoming (mysql_servers_v2) records will be fetched from remote peer. mysql_servers records will be + * @brief mysql_servers_v2 records will be fetched from remote peer. mysql_servers records will be * fetched if fetch_runtime_mysql_servers flag is true. * - * @param peer_mysql_server_incoming checksum and epoch of mysql_servers_incoming from remote peer + * @param peer_mysql_server_v2 checksum and epoch of mysql_servers_v2 from remote peer * @param peer_runtime_mysql_server checksum and epoch of mysql_servers from remote peer * @param fetch_runtime_mysql_servers fetch mysql_servers records if value is true */ -void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers_incoming_checksum_t& peer_mysql_server_incoming, +void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_checksum_t& peer_mysql_server_v2, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, bool fetch_runtime_mysql_servers) { char* hostname = NULL; char* ip_address = NULL; uint16_t port = 0; - char* peer_mysql_servers_incoming_checksum = NULL; + char* peer_mysql_servers_v2_checksum = NULL; char* peer_runtime_mysql_servers_checksum = NULL; - pthread_mutex_lock(&GloProxyCluster->update_mysql_servers_incoming_mutex); - nodes.get_peer_to_sync_mysql_servers_incoming(&hostname, &port, &peer_mysql_servers_incoming_checksum, + pthread_mutex_lock(&GloProxyCluster->update_mysql_servers_v2_mutex); + nodes.get_peer_to_sync_mysql_servers_v2(&hostname, &port, &peer_mysql_servers_v2_checksum, &peer_runtime_mysql_servers_checksum, &ip_address); if (hostname) { char* username = NULL; @@ -1801,7 +1801,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers MYSQL* conn = mysql_init(NULL); if (conn == NULL) { proxy_error("Unable to run mysql_init()\n"); - goto __exit_pull_mysql_servers_incoming_from_peer; + goto __exit_pull_mysql_servers_v2_from_peer; } GloProxyCluster->get_credentials(&username, &password); if (strlen(username)) { // do not monitor if the username is empty @@ -1811,8 +1811,8 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers Incoming from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_mysql_servers_incoming_checksum); - proxy_info("Cluster: Fetching MySQL Servers Incoming from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_mysql_servers_incoming_checksum); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers v2 from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_mysql_servers_v2_checksum); + proxy_info("Cluster: Fetching MySQL Servers v2 from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_mysql_servers_v2_checksum); rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { MySQL_Monitor::dns_cache_update_socket(conn->host, conn->net.fd); @@ -1821,9 +1821,9 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers // servers messages std::string fetch_servers_done = ""; - string_format("Cluster: Fetching 'MySQL Servers Incoming' from peer %s:%d completed\n", fetch_servers_done, hostname, port); + string_format("Cluster: Fetching 'MySQL Servers v2' from peer %s:%d completed\n", fetch_servers_done, hostname, port); std::string fetch_servers_err = ""; - string_format("Cluster: Fetching 'MySQL Servers Incoming' from peer %s:%d failed: \n", fetch_servers_err, hostname, port); + string_format("Cluster: Fetching 'MySQL Servers v2' from peer %s:%d failed: \n", fetch_servers_err, hostname, port); // group_replication_hostgroups messages std::string fetch_group_replication_hostgroups = ""; @@ -1858,7 +1858,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers */ fetch_query queries[] = { { - CLUSTER_QUERY_MYSQL_SERVERS_INCOMING, + CLUSTER_QUERY_MYSQL_SERVERS_V2, p_cluster_counter::pulled_mysql_servers_success, p_cluster_counter::pulled_mysql_servers_failure, { "", fetch_servers_done, fetch_servers_err } @@ -1934,8 +1934,8 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers if (fetching_error == false) { const uint64_t servers_hash = compute_servers_tables_raw_checksum(results, 6); // ignore runtime_mysql_servers in checksum calculation const string computed_checksum{ get_checksum_from_hash(servers_hash) }; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers Incoming from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); - proxy_info("Cluster: Computed checksum for MySQL Servers Incoming from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers v2 from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + proxy_info("Cluster: Computed checksum for MySQL Servers v2 from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); bool runtime_checksum_matches = true; @@ -1947,7 +1947,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers runtime_checksum_matches = (runtime_mysql_server_computed_checksum == peer_runtime_mysql_servers_checksum); } - if (computed_checksum == peer_mysql_servers_incoming_checksum && runtime_checksum_matches == true) { + if (computed_checksum == peer_mysql_servers_v2_checksum && runtime_checksum_matches == true) { // No need to perform the conversion if checksums don't match const incoming_servers_t incoming_servers{ convert_mysql_servers_resultsets(results) }; // we are OK to sync! @@ -2169,13 +2169,13 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers resultset->dump_to_stderr(); delete resultset; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime MySQL Servers Incoming from peer %s:%d\n", hostname, port); - proxy_info("Cluster: Loading to runtime MySQL Servers Incoming from peer %s:%d\n", hostname, port); - GloAdmin->load_mysql_servers_to_runtime(incoming_servers, peer_runtime_mysql_server, peer_mysql_server_incoming); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime MySQL Servers v2 from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Loading to runtime MySQL Servers v2 from peer %s:%d\n", hostname, port); + GloAdmin->load_mysql_servers_to_runtime(incoming_servers, peer_runtime_mysql_server, peer_mysql_server_v2); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Servers Incoming from peer %s:%d\n", hostname, port); - proxy_info("Cluster: Saving to disk MySQL Servers Incoming from peer %s:%d\n", hostname, port); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Servers v2 from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Saving to disk MySQL Servers v2 from peer %s:%d\n", hostname, port); GloAdmin->flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); } else { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); @@ -2183,11 +2183,11 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers } GloAdmin->mysql_servers_wrunlock(); } else { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", - hostname, port, peer_mysql_servers_incoming_checksum, computed_checksum.c_str()); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching MySQL Servers v2 from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, peer_mysql_servers_v2_checksum, computed_checksum.c_str()); proxy_info( - "Cluster: Fetching MySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", - hostname, port, peer_mysql_servers_incoming_checksum, computed_checksum.c_str() + "Cluster: Fetching MySQL Servers v2 from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, peer_mysql_servers_v2_checksum, computed_checksum.c_str() ); metrics.p_counter_array[p_cluster_counter::pulled_mysql_variables_failure]->Increment(); } @@ -2211,7 +2211,7 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers if (password) { free(password); } - __exit_pull_mysql_servers_incoming_from_peer: + __exit_pull_mysql_servers_v2_from_peer: if (conn) { if (conn->net.pvio) { mysql_close(conn); @@ -2222,13 +2222,13 @@ void ProxySQL_Cluster::pull_mysql_servers_incoming_from_peer(const mysql_servers if (ip_address) free(ip_address); - if (peer_mysql_servers_incoming_checksum) - free (peer_mysql_servers_incoming_checksum); + if (peer_mysql_servers_v2_checksum) + free (peer_mysql_servers_v2_checksum); if (peer_runtime_mysql_servers_checksum) free(peer_runtime_mysql_servers_checksum); } - pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_incoming_mutex); + pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_v2_mutex); } void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, const string& expected_checksum, const time_t epoch) { @@ -3006,30 +3006,30 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_runtime_mysql_servers(char **host, } } -void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_incoming(char** host, uint16_t* port, - char** peer_mysql_servers_incoming_checksum, char** peer_runtime_mysql_servers_checksum, char** ip_address) { +void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_v2(char** host, uint16_t* port, + char** peer_mysql_servers_v2_checksum, char** peer_runtime_mysql_servers_checksum, char** ip_address) { unsigned long long version = 0; unsigned long long epoch = 0; unsigned long long max_epoch = 0; char* hostname = NULL; char* ip_addr = NULL; uint16_t p = 0; - char* mysql_servers_incoming_checksum = NULL; + char* mysql_servers_v2_checksum = NULL; char* runtime_mysql_servers_checksum = NULL; //pthread_mutex_lock(&mutex); //unsigned long long curtime = monotonic_time(); unsigned int diff_ms = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, 0); for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry* node = it->second; - ProxySQL_Checksum_Value_2* v = &node->checksums_values.mysql_servers_incoming; + ProxySQL_Checksum_Value_2* v = &node->checksums_values.mysql_servers_v2; if (v->version > 1) { if (v->epoch > epoch) { max_epoch = v->epoch; if (v->diff_check > diff_ms) { epoch = v->epoch; version = v->version; - if (mysql_servers_incoming_checksum) { - free(mysql_servers_incoming_checksum); + if (mysql_servers_v2_checksum) { + free(mysql_servers_v2_checksum); } if (runtime_mysql_servers_checksum) { free(runtime_mysql_servers_checksum); @@ -3040,7 +3040,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_incoming(char** host if (ip_addr) { free(ip_addr); } - mysql_servers_incoming_checksum = strdup(v->checksum); + mysql_servers_v2_checksum = strdup(v->checksum); runtime_mysql_servers_checksum = strdup(node->checksums_values.mysql_servers.checksum); hostname = strdup(node->get_hostname()); const char* ip = node->get_ipaddress(); @@ -3055,14 +3055,14 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_incoming(char** host // pthread_mutex_unlock(&mutex); if (epoch) { if (max_epoch > epoch) { - proxy_warning("Cluster: detected a peer with mysql_servers_incoming epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + proxy_warning("Cluster: detected a peer with mysql_servers_v2 epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); if (hostname) { free(hostname); hostname = NULL; } - if (mysql_servers_incoming_checksum) { - free(mysql_servers_incoming_checksum); - mysql_servers_incoming_checksum = NULL; + if (mysql_servers_v2_checksum) { + free(mysql_servers_v2_checksum); + mysql_servers_v2_checksum = NULL; } if (runtime_mysql_servers_checksum) { free(runtime_mysql_servers_checksum); @@ -3078,10 +3078,10 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_incoming(char** host *host = hostname; *port = p; *ip_address = ip_addr; - *peer_mysql_servers_incoming_checksum = mysql_servers_incoming_checksum; + *peer_mysql_servers_v2_checksum = mysql_servers_v2_checksum; *peer_runtime_mysql_servers_checksum = runtime_mysql_servers_checksum; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu, mysql_servers_incoming checksum %s, runtime_mysql_servers %s\n", hostname, p, version, epoch, mysql_servers_incoming_checksum, runtime_mysql_servers_checksum); - proxy_info("Cluster: detected peer %s:%d with mysql_servers_incoming version %llu, epoch %llu\n", hostname, p, version, epoch); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, mysql_servers_v2 checksum %s, runtime_mysql_servers %s\n", hostname, p, version, epoch, mysql_servers_v2_checksum, runtime_mysql_servers_checksum); + proxy_info("Cluster: detected peer %s:%d with mysql_servers_v2 version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -3393,7 +3393,7 @@ SQLite3_result * ProxySQL_Cluster_Nodes::stats_proxysql_servers_checksums() { vals[3] = &node->checksums_values.mysql_users; vals[4] = &node->checksums_values.mysql_variables; vals[5] = &node->checksums_values.proxysql_servers; - vals[6] = &node->checksums_values.mysql_servers_incoming; + vals[6] = &node->checksums_values.mysql_servers_v2; for (int i=0; i<7 ; i++) { ProxySQL_Checksum_Value_2 *v = vals[i]; char **pta=(char **)malloc(sizeof(char *)*colnum); @@ -3421,7 +3421,7 @@ SQLite3_result * ProxySQL_Cluster_Nodes::stats_proxysql_servers_checksums() { pta[2]=strdup((char *)"proxysql_servers"); break; case 6: - pta[2] = strdup((char*)"mysql_servers_incoming"); + pta[2]=strdup((char*)"mysql_servers_v2"); break; default: break; @@ -3547,7 +3547,7 @@ vector> get_module_checksums(ProxySQL_N res.push_back({"mysql_users", &entry->checksums_values.mysql_users}); res.push_back({"mysql_variables", &entry->checksums_values.mysql_variables}); res.push_back({"proxysql_servers", &entry->checksums_values.proxysql_servers}); - res.push_back({"mysql_servers_incoming", &entry->checksums_values.mysql_servers_incoming}); + res.push_back({"mysql_servers_v2", &entry->checksums_values.mysql_servers_v2}); return res; } @@ -4182,7 +4182,7 @@ ProxySQL_Cluster::ProxySQL_Cluster() : proxysql_servers_to_monitor(NULL) { pthread_mutex_init(&mutex,NULL); pthread_mutex_init(&update_mysql_query_rules_mutex,NULL); pthread_mutex_init(&update_runtime_mysql_servers_mutex,NULL); - pthread_mutex_init(&update_mysql_servers_incoming_mutex, NULL); + pthread_mutex_init(&update_mysql_servers_v2_mutex, NULL); pthread_mutex_init(&update_mysql_users_mutex,NULL); pthread_mutex_init(&update_proxysql_servers_mutex,NULL); pthread_mutex_init(&update_mysql_variables_mutex,NULL); diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index 65dd99907e..911c2075ae 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -385,7 +385,7 @@ uint64_t ProxySQL_GlobalVariables::generate_global_checksum() { myhash.Update(v->checksum,strlen(v->checksum)); myhash.Update(&v->version,sizeof(v->version)); } - v = &checksums_values.mysql_servers_incoming; + v = &checksums_values.mysql_servers_v2; if (v->version) { myhash.Update(v->checksum, strlen(v->checksum)); myhash.Update(&v->version, sizeof(v->version)); From 1af0586d117c2e3dd6d632b0b62e22b8a9fe3cc8 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Tue, 11 Apr 2023 01:06:43 +0500 Subject: [PATCH 06/56] * Added saving mysql_servers to disk. * Added detailed comments. --- lib/ProxySQL_Cluster.cpp | 42 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 1fddef4749..62e0593a62 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1669,6 +1669,13 @@ incoming_servers_t convert_mysql_servers_resultsets(const std::vectorservers_add(runtime_mysql_servers_resultset.get()); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Updating runtime_mysql_servers from peer %s:%d", hostname, port); MyHGM->commit(runtime_mysql_servers_resultset.release(), peer_runtime_mysql_server, nullptr, {}, true); + + if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving Runtime MySQL Servers to Database\n"); + GloAdmin->save_mysql_servers_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Servers v2 from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Saving to disk MySQL Servers v2 from peer %s:%d\n", hostname, port); + GloAdmin->flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); + } GloAdmin->mysql_servers_wrunlock(); // free result @@ -1775,12 +1790,31 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ } /** - * @brief mysql_servers_v2 records will be fetched from remote peer. mysql_servers records will be - * fetched if fetch_runtime_mysql_servers flag is true. + * @brief mysql_servers_v2 records will be fetched from remote peer. mysql_servers records will be fetched if + * fetch_runtime_mysql_servers flag is true. + * + * @details The previous implementation of the "pull_mysql_servers_from_peer" method fetched data from "mysql_servers" (equivalent to runtime mysql_servers) + * and other dependent modules like "mysql_replication_hostgroups", "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", + * "mysql_aws_aurora_hostgroups", and "mysql_hostgroup_attributes". It then computed an accumulated checksum and compares it with the + * peer checksum. If they matched, the configuration was loaded and saved to disk if "cluster_mysql_servers_save_to_disk" was set to true. + * + * The new implementation, "pull_mysql_servers_v2_from_peer", instead fetches data from "mysql_servers_v2" (equivalent to admin mysql_servers) + * and the same dependent modules. It then computes an accumulated checksum and compares it with the peer checksum. If they matched, the + * configuration was loaded and saved to disk if "cluster_mysql_servers_save_to_disk" was set to true. Additionally, if the "fetch_runtime_mysql_servers" + * option is enabled (if cluster_mysql_servers_sync_algorithm value is set to 1), the "mysql_servers" table will also be fetched and its checksum will be + * computed and matched with the peer checksum. If they match, the configuration will be loaded and saved to disk if the "cluster_mysql_servers_save_to_disk" + * option is true. + * + * Apart from separately fetching the runtime mysql_servers, the primary distinction between the previous and new implementations lies in the + * fetching of different tables (mysql_servers vs mysql_servers_v2) and computing of checksum. In the previous version, + * the checksum for "mysql_servers" was computed and added to the checksums of other dependent modules. In contrast, the new version + * calculates the checksum for "mysql_servers_v2" and combines it with the checksums of other dependent modules. * * @param peer_mysql_server_v2 checksum and epoch of mysql_servers_v2 from remote peer * @param peer_runtime_mysql_server checksum and epoch of mysql_servers from remote peer * @param fetch_runtime_mysql_servers fetch mysql_servers records if value is true + * + * NOTE: pull_mysql_servers_v2_from_peer will always be called irrespective of cluster_mysql_servers_sync_algorithm value. */ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_checksum_t& peer_mysql_server_v2, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, bool fetch_runtime_mysql_servers) { @@ -2174,6 +2208,10 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch GloAdmin->load_mysql_servers_to_runtime(incoming_servers, peer_runtime_mysql_server, peer_mysql_server_v2); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { + if (fetch_runtime_mysql_servers == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving Runtime MySQL Servers to Database\n"); + GloAdmin->save_mysql_servers_runtime_to_database(false); + } proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk MySQL Servers v2 from peer %s:%d\n", hostname, port); proxy_info("Cluster: Saving to disk MySQL Servers v2 from peer %s:%d\n", hostname, port); GloAdmin->flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); From bacf3a728eedb0cd3d981aae32283360eceb2606 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Tue, 18 Apr 2023 12:24:26 +0500 Subject: [PATCH 07/56] * The code block responsible for syncing admin variables has been moved ahead of the 'mysql_servers' code block. This ensures that the most recent value of 'admin-cluster_mysql_servers_sync_algorithm' is available in mysql_servers sync. * Reset global checksum if 'admin-cluster_mysql_servers_sync_algorithm' is changed. --- lib/MySQL_HostGroups_Manager.cpp | 5 +- lib/ProxySQL_Admin.cpp | 10 ++-- lib/ProxySQL_Cluster.cpp | 93 ++++++++++++++++---------------- 3 files changed, 56 insertions(+), 52 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index ac2319cd6e..8dd1d47558 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2319,7 +2319,10 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru /** * @brief Computes checksum for the admin mysql_server resultset and generates an accumulated checksum by including the following modules: * MYSQL_REPLICATION_HOSTGROUPS, MYSQL_GROUP_REPLICATION_HOSTGROUPS, MYSQL_GALERA_HOSTGROUPS, and MYSQL_HOSTGROUP_ATTRIBUTES. - * + * + * @details The 'incoming_mysql_servers_v2' includes the same records as 'main.mysql_servers'. If this set of records is empty, then the + * records are retrieved from 'main.mysql_servers'. + * * @param incoming_mysql_servers_v2 resultset of admin mysql_servers or can be a nullptr. */ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2) { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index d09511671d..5b674ff468 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -8223,12 +8223,14 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (!strcasecmp(name,"cluster_mysql_servers_sync_algorithm")) { int intv=atoi(value); if (intv >= 1 && intv <= 3) { - variables.cluster_mysql_servers_sync_algorithm =intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_sync_algorithm, intv); - proxy_info("'cluster_mysql_servers_sync_algorithm' updated. Resetting global checksums to force Cluster re-sync."); - GloProxyCluster->Reset_Global_Checksums(lock); + if (variables.cluster_mysql_servers_sync_algorithm != intv) { + proxy_info("'cluster_mysql_servers_sync_algorithm' updated. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); + } + variables.cluster_mysql_servers_sync_algorithm=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_sync_algorithm, intv); return true; } else { return false; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 62e0593a62..a49e4789ab 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -696,14 +696,46 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { // we now do a series of checks, and we take action // note that this is done outside the critical section // as mutex on GloVars.checksum_mutex is already released + unsigned int diff_av = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_admin_variables_diffs_before_sync,0); unsigned int diff_mqr = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync,0); unsigned int diff_ms = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync,0); unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_users_diffs_before_sync,0); unsigned int diff_ps = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync,0); unsigned int diff_mv = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync,0); unsigned int diff_lv = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync,0); - unsigned int diff_av = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_admin_variables_diffs_before_sync,0); ProxySQL_Checksum_Value_2 *v = NULL; + if (diff_av) { + v = &checksums_values.admin_variables; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.version, 0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.epoch, 0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.checksum, 0); + const string expected_checksum { v->checksum }; + + if (v->version > 1) { + if ( + (own_version == 1) // we just booted + || + (v->epoch > own_epoch) // epoch is newer + ) { + if (v->diff_check >= diff_av) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_global_variables_from_peer("admin", expected_checksum, v->epoch); + } + } + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_av*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av * 10)); + proxy_error("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_admin_variables_share_epoch]->Increment(); + } + } else { + if (v->diff_check && (v->diff_check % (diff_av*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av * 10)); + proxy_warning("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_admin_variables_version_one]->Increment(); + } + } + } if (diff_mqr) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_query_rules.version,0); unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_query_rules.epoch,0); @@ -760,20 +792,19 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - if (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_v2) { - ProxySQL_Checksum_Value_2* runtime_mysql_server_checksum = &checksums_values.mysql_servers; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_v2 and mysql_servers from peer %s:%d", hostname, port); - // fetching mysql server v2 and runtime mysql server - GloProxyCluster->pull_mysql_servers_v2_from_peer({ v->checksum, v->epoch }, - { runtime_mysql_server_checksum->checksum, runtime_mysql_server_checksum->epoch }, true); - - runtime_mysql_servers_already_loaded = true; - } else { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching mysql_servers_v2 from peer %s:%d", hostname, port); - // fetching mysql server v2 - // to avoid runtime_mysql_servers generating new epoch, we are passing mysql_server_v2 checksum and epoch - GloProxyCluster->pull_mysql_servers_v2_from_peer({ v->checksum, v->epoch }, { v->checksum, v->epoch }); - } + ProxySQL_Checksum_Value_2* runtime_mysql_server_checksum = &checksums_values.mysql_servers; + + const bool fetch_runtime = (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_v2); + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetch mysql_servers_v2:'YES', mysql_servers:'%s' from peer %s:%d\n", (fetch_runtime ? "YES" : "NO"), + hostname, port); + proxy_info("Cluster: Fetch mysql_servers_v2:'YES', mysql_servers:'%s' from peer %s:%d\n", (fetch_runtime ? "YES" : "NO"), + hostname, port); + + GloProxyCluster->pull_mysql_servers_v2_from_peer({ v->checksum, v->epoch }, + { runtime_mysql_server_checksum->checksum, runtime_mysql_server_checksum->epoch }, fetch_runtime); + + runtime_mysql_servers_already_loaded = fetch_runtime; } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms * 10)) == 0)) { @@ -888,38 +919,6 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } } } - if (diff_av) { - v = &checksums_values.admin_variables; - unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.version, 0); - unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.epoch, 0); - char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.checksum, 0); - const string expected_checksum { v->checksum }; - - if (v->version > 1) { - if ( - (own_version == 1) // we just booted - || - (v->epoch > own_epoch) // epoch is newer - ) { - if (v->diff_check >= diff_av) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("admin", expected_checksum, v->epoch); - } - } - if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_av*10)) == 0)) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av * 10)); - proxy_error("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_admin_variables_share_epoch]->Increment(); - } - } else { - if (v->diff_check && (v->diff_check % (diff_av*10)) == 0) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av * 10)); - proxy_warning("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_admin_variables_version_one]->Increment(); - } - } - } if (GloMyLdapAuth && diff_lv) { v = &checksums_values.ldap_variables; unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.version, 0); From 625675ddbd7fe0a5071030ccb69e434c1e94372e Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Tue, 18 Apr 2023 12:46:43 +0500 Subject: [PATCH 08/56] Created TAP test to ensure the cluster synchronization of 'runtime_mysql_servers' and 'mysql_servers_v2' across all nodes. The test involves creating two replica nodes, one with a monitor enabled and the other with a monitor disabled. The test verifies that the correct records are synced among these nodes based on the value of 'admin-cluster_mysql_servers_sync_algorithm'. --- .../test_cluster_sync_nomonitor/.gitignore | 4 + .../test_cluster_sync_withmonitor/.gitignore | 4 + .../test_cluster_sync_mysql_servers-t.cpp | 971 ++++++++++++++++++ 3 files changed, 979 insertions(+) create mode 100644 test/tap/tests/test_cluster_sync_config/test_cluster_sync_nomonitor/.gitignore create mode 100644 test/tap/tests/test_cluster_sync_config/test_cluster_sync_withmonitor/.gitignore create mode 100644 test/tap/tests/test_cluster_sync_mysql_servers-t.cpp diff --git a/test/tap/tests/test_cluster_sync_config/test_cluster_sync_nomonitor/.gitignore b/test/tap/tests/test_cluster_sync_config/test_cluster_sync_nomonitor/.gitignore new file mode 100644 index 0000000000..25cc06f021 --- /dev/null +++ b/test/tap/tests/test_cluster_sync_config/test_cluster_sync_nomonitor/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory + +# Except this file +!.gitignore \ No newline at end of file diff --git a/test/tap/tests/test_cluster_sync_config/test_cluster_sync_withmonitor/.gitignore b/test/tap/tests/test_cluster_sync_config/test_cluster_sync_withmonitor/.gitignore new file mode 100644 index 0000000000..25cc06f021 --- /dev/null +++ b/test/tap/tests/test_cluster_sync_config/test_cluster_sync_withmonitor/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory + +# Except this file +!.gitignore \ No newline at end of file diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp new file mode 100644 index 0000000000..25d5b09672 --- /dev/null +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -0,0 +1,971 @@ +/** + * @file test_cluster_sync_mysql_servers-t.cpp + * @brief Checks that ProxySQL mysql_server and mysql_server_v2 is properly syncing. + * @details Checks the sync of the following tables: + * - 'mysql_servers_v2' + * - 'mysql_servers' + * + * + * Test Cluster Isolation: + * ---------------------- + * For guaranteeing that this test doesn't invalidate the configuration of a running ProxySQL cluster and + * that after the test, the previous valid configuration is restored, the following actions are performed: + * + * 1. The Core nodes from the current cluster configuration are backup. + * 2. Primary (currently tested instance) is removed from the Core nodes. + * 3. A sync wait until all core nodes have performed the removal of primary is executed. + * 4. Now Primary is isolated from the previous cluster, tests can proceed. Primary is setup to hold itself + * in its 'proxysql_servers' as well as the target spawned replica. + * 5. After the tests recover the primary configuration and add it back to the Core nodes from Cluster: + * - Recover the previous 'mysql_servers' from disk, and load them to runtime, discarding any previous + * config performed during the test. + * - Insert the primary back into a Core node from cluster and wait for all nodes to sync including it. + * - Insert into the primary the previous backup Core nodes from Cluster and load to runtime. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +#define MYSQL_QUERY__(mysql, query) \ + do { \ + if (mysql_query(mysql, query)) { \ + fprintf(stderr, "File %s, line %d, Error: %s\n", \ + __FILE__, __LINE__, mysql_error(mysql)); \ + goto cleanup; \ + } \ + } while(0) + +// GLOBAL TEST PARAMETERS +const uint32_t SYNC_TIMEOUT = 10; +const uint32_t CONNECT_TIMEOUT = 10; +const uint32_t R_NOMONITOR_PORT = 96061; +const uint32_t R_WITHMONITOR_PORT = 96062; + +const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; + +using mysql_server_tuple = std::tuple; +using replication_hostgroups_tuple = std::tuple; + +/** + * @brief Computes the checksum for the resultset, excluding records labeled as 'OFFLINE_HARD', instead of checking each row individually. + * + * @param resultset mysql_servers + * + */ +uint64_t mysql_servers_raw_checksum(MYSQL_RES* resultset) { + if (resultset == nullptr) { return 0; } + + uint64_t num_rows = mysql_num_rows(resultset); + if (num_rows == 0) { return 0; } + + MYSQL_FIELD* fields = mysql_fetch_fields(resultset); + uint32_t num_fields = mysql_num_fields(resultset); + uint32_t status_idx = 0; + + for (uint32_t i = 0; i < num_fields; i++) { + if (strcmp(fields[i].name, "status") == 0) { + status_idx = i; + } + } + + SpookyHash myhash {}; + myhash.Init(19,3); + + while (MYSQL_ROW row = mysql_fetch_row(resultset)) { + for (uint32_t i = 0; i < num_fields; i++) { + if (strcmp(row[status_idx], "OFFLINE_HARD") == 0) { + continue; + } + + if (row[i]) { + if (strcmp(fields[i].name, "status") == 0) { + if (strcmp(row[i], "ONLINE") == 0 || strcmp(row[i], "SHUNNED") == 0) { + myhash.Update("0", strlen("0")); + } else { + myhash.Update("2", strlen("1")); + } + } else { + // computing 'strlen' is required see @details + myhash.Update(row[i], strlen(row[i])); + } + } else { + myhash.Update("", 0); + } + } + } + + // restore the initial resulset index + mysql_data_seek(resultset, 0); + + uint64_t res_hash = 0, hash2 = 0; + myhash.Final(&res_hash, &hash2); + + return res_hash; +} + + +/** + * @brief Helper function to verify that the sync of a table (or variable) have been performed. + * + * @param r_proxy_admin An already opened connection to ProxySQL. + * @param queries Queries to be executed that should return a **non-zero** value after the sync has taken place. + * @param sync_timeout Timeout for the sync to happen. + * + * @return EXIT_SUCCESS in case of success, otherwise: + * - '-1' if a query against Admin fails to be performed (failure is logged). + * - '-2' if timeout expired without sync being completed. + */ +int sync_checker(MYSQL* r_proxy_admin, const std::vector& queries, uint32_t sync_timeout) { + bool not_synced_query = false; + uint waited = 0; + + while (waited < sync_timeout) { + not_synced_query = false; + + // Check that all the entries have been synced + for (const auto& query : queries) { + int q_res = mysql_query(r_proxy_admin, query.c_str()); + if (q_res != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxy_admin)); + return -1; + } + + MYSQL_RES* proxysql_servers_res = mysql_store_result(r_proxy_admin); + MYSQL_ROW row = mysql_fetch_row(proxysql_servers_res); + int row_value = atoi(row[0]); + mysql_free_result(proxysql_servers_res); + + if (row_value == 0) { + not_synced_query = true; + break; + } + } + + if (not_synced_query) { + waited += 1; + sleep(1); + } else { + break; + } + } + + if (not_synced_query) { + return -2; + } else { + return EXIT_SUCCESS; + } +} + +int check_nodes_sync( + const CommandLine& cl, const std::vector& core_nodes, const std::string& check_query, uint32_t sync_timeout +) { + for (const auto& node : core_nodes) { + const std::string host { node[0] }; + const int port = std::stol(node[1]); + + MYSQL* c_node_admin = mysql_init(NULL); + if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(c_node_admin)); + return EXIT_FAILURE; + } + + int not_synced = sync_checker(c_node_admin, { check_query }, sync_timeout); + if (not_synced != EXIT_SUCCESS) { + const std::string err_msg { "Node '" + host + ":" + std::to_string(port) + "' sync timed out" }; + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str()); + return EXIT_FAILURE; + } + } + + return EXIT_SUCCESS; +} + +int insert_mysql_servers_records(MYSQL* proxy_admin, const std::vector& insert_mysql_servers_values, + const std::vector& insert_replication_hostgroups_values) { + + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_servers"); + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups"); + + // Configure 'mysql_servers' and check sync with NULL comments + const char* t_insert_mysql_servers = + "INSERT INTO mysql_servers (" + " hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections," + " max_replication_lag, use_ssl, max_latency_ms, comment" + ") VALUES (%d, '%s', %d, %d, '%s', %d, %d, %d, %d, %d, %d, '%s')"; + + const char* t_mysql_replication_hostgroups = + "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (%d,%d,'%s')"; + + for (auto const& values : insert_mysql_servers_values) { + std::string insert_mysql_servers_hostgroup_query; + string_format( + t_insert_mysql_servers, + insert_mysql_servers_hostgroup_query, + std::get<0>(values), + std::get<1>(values).c_str(), + std::get<2>(values), + std::get<3>(values), + std::get<4>(values).c_str(), + std::get<5>(values), + std::get<6>(values), + std::get<7>(values), + std::get<8>(values), + std::get<9>(values), + std::get<10>(values), + std::get<11>(values).c_str() + ); + + // Insert the new mysql_servers hostgroups values + MYSQL_QUERY(proxy_admin, insert_mysql_servers_hostgroup_query.c_str()); + } + + for (auto const& values : insert_replication_hostgroups_values) { + std::string insert_mysql_replication_hostgroups_query; + string_format( + t_mysql_replication_hostgroups, + insert_mysql_replication_hostgroups_query, + std::get<0>(values), + std::get<1>(values), + std::get<2>(values).c_str() + ); + + // Insert the new mysql_replication_hostgroups values + MYSQL_QUERY(proxy_admin, insert_mysql_replication_hostgroups_query.c_str()); + } + + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + return EXIT_SUCCESS; +} + +int wait_for_node_sync(MYSQL* r_proxy_admin, uint64_t master_checksum, const std::string& table) { + uint waited = 0; + bool not_synced = false; + const std::string& query = "SELECT * FROM " + table; + + while (waited < SYNC_TIMEOUT) { + not_synced = false; + + MYSQL_QUERY(r_proxy_admin, query.c_str()); + MYSQL_RES* mysql_servers_res = mysql_store_result(r_proxy_admin); + auto replica_checksum = mysql_servers_raw_checksum(mysql_servers_res); + mysql_free_result(mysql_servers_res); + + if (replica_checksum != master_checksum) { + not_synced = true; + diag("Waiting for '%s' to be synced", table.c_str()); + } + + if (not_synced) { + waited += 1; + sleep(1); + } else { + break; + } + } + + if (not_synced) { + diag("'wait_for_node_sync' timeout for query '%s'", query.c_str()); + } + + return not_synced; +}; + +int check_mysql_servers_sync( + const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_withmonitor_admin, MYSQL* r_proxy_nomonitor_admin, + int cluster_sync_mysql_servers_algorithm +) { + std::string print_master_mysql_servers_hostgroups; + string_format(t_debug_query, print_master_mysql_servers_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM mysql_servers"); + std::string print_master_runtime_mysql_servers_hostgroups; + string_format(t_debug_query, print_master_runtime_mysql_servers_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_servers"); + + std::string print_nomonitor_replica_mysql_servers_hostgroups; + string_format(t_debug_query, print_nomonitor_replica_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_NOMONITOR_PORT, "SELECT * FROM mysql_servers"); + std::string print_nomonitor_replica_runtime_mysql_servers_hostgroups; + string_format(t_debug_query, print_nomonitor_replica_runtime_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_NOMONITOR_PORT, "SELECT * FROM runtime_mysql_servers"); + + std::string print_withmonitor_replica_mysql_servers_hostgroups; + string_format(t_debug_query, print_withmonitor_replica_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_WITHMONITOR_PORT, "SELECT * FROM mysql_servers"); + std::string print_withmonitor_replica_runtime_mysql_servers_hostgroups; + string_format(t_debug_query, print_withmonitor_replica_runtime_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_WITHMONITOR_PORT, "SELECT * FROM runtime_mysql_servers"); + + std::cout << "MASTER 'MYSQL SERVERS' TABLE BEFORE SYNC:" << std::endl; + system(print_master_mysql_servers_hostgroups.c_str()); + + // Wait till read_only actions have been performed + std::string monitor_read_only_interval; + int g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_interval", monitor_read_only_interval); + if (g_err) { return EXIT_FAILURE; } + std::string monitor_read_only_timeout {}; + g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_timeout", monitor_read_only_timeout); + if (g_err) { return EXIT_FAILURE; } + + uint64_t wait = std::stol(monitor_read_only_interval) + std::stol(monitor_read_only_timeout); + usleep((wait*1000)*2); + + std::cout << "MASTER 'RUNTIME MYSQL SERVERS' TABLE BEFORE SYNC:" << std::endl; + system(print_master_runtime_mysql_servers_hostgroups.c_str()); + + + uint64_t master_mysql_servers_checksum = 0; + uint64_t master_runtime_mysql_servers_checksum = 0; + + // fetch master mysql_servers resultset and compute it's hash + { + MYSQL_QUERY(proxy_admin, "SELECT * FROM mysql_servers"); + MYSQL_RES* mysql_servers_res = mysql_store_result(proxy_admin); + master_mysql_servers_checksum = mysql_servers_raw_checksum(mysql_servers_res); + mysql_free_result(mysql_servers_res); + } + + // fetch master runtime_mysql_servers resultset and compute it's hash + { + MYSQL_QUERY(proxy_admin, "SELECT * FROM runtime_mysql_servers"); + MYSQL_RES* mysql_servers_res = mysql_store_result(proxy_admin); + master_runtime_mysql_servers_checksum = mysql_servers_raw_checksum(mysql_servers_res); + mysql_free_result(mysql_servers_res); + } + + // SYNCH CHECK + bool not_synced_query = false; + + if (cluster_sync_mysql_servers_algorithm == 1) { + + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + } else if (cluster_sync_mysql_servers_algorithm == 2) { + + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + } else if (cluster_sync_mysql_servers_algorithm == 3) { + + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + } + + return EXIT_SUCCESS; +} + +/** + * @brief Assumes that 'proxysql_servers' holds at least the one entry required for this test. + * @details It's assumed that primary ProxySQL is part of a Cluster. + */ +int update_proxysql_servers(const CommandLine& cl, MYSQL* admin) { + const char update_proxysql_servers_t[] { + "UPDATE proxysql_servers SET comment='%s' WHERE hostname='%s' and port=%d" + }; + + cfmt_t update_servers { + cstr_format(update_proxysql_servers_t, std::to_string(time(NULL)).c_str(), cl.host, cl.admin_port) + }; + MYSQL_QUERY_T(admin, update_servers.str.c_str()); + MYSQL_QUERY_T(admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); + + return EXIT_SUCCESS; +} + +int setup_config_file(const CommandLine& cl, uint32_t r_port, const std::string& config_filename) { + + const std::string& workdir = std::string(cl.workdir); + + const std::string& t_fmt_config_file = workdir + "test_cluster_sync_config/test_cluster_sync-t.cnf"; + const std::string& fmt_config_file = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/test_cluster_sync.cnf"; + const std::string& datadir_path = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename; + + // Prepare the configuration file + config_t cfg {}; + + config_init(&cfg); + + if (!config_read_file(&cfg, t_fmt_config_file.c_str())) { + fprintf(stderr, "%s:%d - %s\n", config_error_file(&cfg), config_error_line(&cfg), config_error_text(&cfg)); + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - Error reading config file."); + config_destroy(&cfg); + return -1; + } + + config_setting_t* r_datadir = config_lookup(&cfg, "datadir"); + if (r_datadir == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - 'datadir' setting not found."); + return -1; + } + + config_setting_t* r_admin_vars = config_lookup(&cfg, "admin_variables"); + if (r_admin_vars == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - 'admin_variables' setting not found."); + return -1; + } + + config_setting_t* r_mysql_ifaces = config_setting_get_member(r_admin_vars, "mysql_ifaces"); + if (r_mysql_ifaces == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - 'mysql_ifaces' setting not found."); + return -1; + } + + int r_ifaces_res = config_setting_set_string(r_mysql_ifaces, std::string { "0.0.0.0:" + std::to_string(r_port) }.c_str()); + if (r_ifaces_res == CONFIG_FALSE) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - Error while trying to set the values for 'mysql_ifaces'."); + return -1; + } + + config_setting_t* p_servers = config_lookup(&cfg, "proxysql_servers"); + if (p_servers == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - 'proxysql_servers' setting not found."); + return -1; + } + + int r_datadir_res = config_setting_set_string(r_datadir, datadir_path.c_str()); + if (r_datadir_res == CONFIG_FALSE) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - Error while trying to set the 'datadir' value."); + return -1; + } + + // Get first group settings + config_setting_t* r_pserver_group = config_setting_get_elem(p_servers, 0); + if (r_pserver_group == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - 'proxysql_servers' doesn't contains first group."); + return -1; + } + config_setting_t* r_pserver_hostname = config_setting_get_member(r_pserver_group, "hostname"); + config_setting_t* r_pserver_port = config_setting_get_member(r_pserver_group, "port"); + + // Check the group members + if (r_pserver_hostname == nullptr || r_pserver_port == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - 'proxysql_servers' doesn't contains the necessary group members."); + return -1; + } + + int fhost_res = config_setting_set_string(r_pserver_hostname, cl.host); + int fport_res = config_setting_set_int(r_pserver_port, cl.admin_port); + + if (fhost_res == CONFIG_FALSE || fport_res == CONFIG_FALSE) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - Error while trying to set the values from env variables."); + return -1; + } + + // Write the new config file + if (config_write_file(&cfg, fmt_config_file.c_str()) == CONFIG_FALSE) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Config file - Error while trying to write the new config file."); + return -1; + } + + config_destroy(&cfg); + + return 0; +} + +int launch_proxysql_replica(const CommandLine& cl, uint32_t r_port, const std::string config_filename, bool monitor_enabled, + const std::atomic& save_proxy_stderr) { + + // Setup the config file using the env variables in 'CommandLine' + if (setup_config_file(cl, r_port, config_filename)) { + return EXIT_FAILURE; + } + + const std::string& workdir = std::string(cl.workdir); + + const std::string& replica_stderr = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/cluster_sync_node_stderr.txt"; + const std::string& proxysql_db = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/proxysql.db"; + const std::string& stats_db = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/proxysql_stats.db"; + const std::string& fmt_config_file = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/test_cluster_sync.cnf"; + + std::string proxy_stdout {}; + std::string proxy_stderr {}; + const std::string& proxy_binary_path = workdir + "../../../src/proxysql"; + + const std::string& proxy_command = proxy_binary_path + " -f " + (monitor_enabled == false ? "-M" : "") + " -c " + fmt_config_file + " > " + replica_stderr + " 2>&1"; + + diag("Launching replica ProxySQL [%s] via 'system' with command : `%s`", config_filename.c_str(), proxy_command.c_str()); + int exec_res = system(proxy_command.c_str()); + + ok(exec_res == 0, "proxysql cluster node [%s] should execute and shutdown nicely. 'system' result was: %d", config_filename.c_str(), exec_res); + + // In case of error place in log the reason + if (exec_res || save_proxy_stderr.load()) { + if (exec_res) { + diag("LOG: Proxysql cluster node [%s] execution failed, logging stderr into 'test_cluster_sync_node_stderr_%s.txt", config_filename.c_str(), config_filename.c_str()); + } else { + diag("LOG: One of the tests failed to pass, logging stderr 'test_cluster_sync_node_stderr_%s.txt", config_filename.c_str()); + } + } + + remove(proxysql_db.c_str()); + remove(stats_db.c_str()); + + return EXIT_SUCCESS; +} + +int main(int, char**) { + int res = 0; + CommandLine cl; + std::atomic save_proxy_stderr(false); + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + + plan(32); + + MYSQL* proxy_admin = mysql_init(NULL); + + // Initialize connections + if (!proxy_admin) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + return EXIT_FAILURE; + } + + // Connnect to local proxysql + if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + return EXIT_FAILURE; + } + + const std::string t_update_proxysql_servers { + "INSERT INTO proxysql_servers (hostname, port, weight, comment) VALUES ('%s', %d, 0, 'proxysql')" + }; + + std::string update_proxysql_servers; + string_format(t_update_proxysql_servers, update_proxysql_servers, cl.host, cl.admin_port); + + // 1. Backup the Core nodes from current cluster configuration + MYSQL_QUERY(proxy_admin, "DROP TABLE IF EXISTS proxysql_servers_sync_test_backup_2687"); + MYSQL_QUERY(proxy_admin, "CREATE TABLE proxysql_servers_sync_test_backup_2687 AS SELECT * FROM proxysql_servers"); + + // 2. Remove primary from Core nodes + MYSQL_QUERY(proxy_admin, "DELETE FROM proxysql_servers WHERE hostname=='127.0.0.1' AND PORT==6032"); + MYSQL_QUERY(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "SELECT hostname,port FROM proxysql_servers"); + MYSQL_RES* my_res = mysql_store_result(proxy_admin); + std::vector core_nodes { extract_mysql_rows(my_res) }; + mysql_free_result(my_res); + + // 3. Wait for all Core nodes to sync (confirm primary out of core nodes) + std::string check_no_primary_query {}; + string_format( + "SELECT CASE COUNT(*) WHEN 0 THEN 1 ELSE 0 END FROM proxysql_servers WHERE hostname=='%s' AND port==%d", + check_no_primary_query, cl.host, cl.admin_port + ); + + int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + // 4. Remove all current servers from primary instance (only secondary sync matters) + MYSQL_QUERY(proxy_admin, "DELETE FROM proxysql_servers"); + MYSQL_QUERY(proxy_admin, update_proxysql_servers.c_str()); + MYSQL_QUERY(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); + + + // Launch proxysql with cluster config and monitor feature disabled + std::thread proxysql_replica_nomonitor_thd(launch_proxysql_replica, std::ref(cl), R_NOMONITOR_PORT, "nomonitor", false, std::ref(save_proxy_stderr)); + + // Launch proxysql with cluster config - with -M commandline + std::thread proxysql_replica_withmonitor_thd(launch_proxysql_replica, std::ref(cl), R_WITHMONITOR_PORT, "withmonitor", true, std::ref(save_proxy_stderr)); + + MYSQL* r_proxysql_nomonitor_admin = NULL; + MYSQL* r_proxysql_withmonitor_admin = NULL; + { + // Waiting for proxysql to be ready + conn_opts_t conn_opts_nomonitor {}; + conn_opts_nomonitor.host = cl.host; + conn_opts_nomonitor.user = "radmin"; + conn_opts_nomonitor.pass = "radmin"; + conn_opts_nomonitor.port = R_NOMONITOR_PORT; + + // connect to proxsqyl replica [nomonitor] + r_proxysql_nomonitor_admin = wait_for_proxysql(conn_opts_nomonitor, CONNECT_TIMEOUT); + + // Once the thread is spanwed we should always go to cleanup to wait + ok(r_proxysql_nomonitor_admin != nullptr, "New instance of proxysql [nomonitor] with cluster config should be properly spawned."); + + if (r_proxysql_nomonitor_admin == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxysql_nomonitor_admin)); + res = -1; + goto cleanup; + } + + conn_opts_t conn_opts_withmonitor {}; + conn_opts_withmonitor.host = cl.host; + conn_opts_withmonitor.user = "radmin"; + conn_opts_withmonitor.pass = "radmin"; + conn_opts_withmonitor.port = R_WITHMONITOR_PORT; + + // connect to proxsqyl replica [nomonitor] + r_proxysql_withmonitor_admin = wait_for_proxysql(conn_opts_withmonitor, CONNECT_TIMEOUT); + + // Once the thread is spanwed we should always go to cleanup to wait + ok(r_proxysql_withmonitor_admin != nullptr, "New instance of proxysql [withmonitor] with cluster config should be properly spawned."); + + if (r_proxysql_withmonitor_admin == nullptr) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxysql_withmonitor_admin)); + res = -1; + goto cleanup; + } + + // setting read_only variables and enabling monitor + MYSQL_QUERY(proxy_admin, "SET mysql-monitor_read_only_interval=200"); + MYSQL_QUERY(proxy_admin, "SET mysql-monitor_read_only_timeout=100"); + MYSQL_QUERY(proxy_admin, "SET mysql-monitor_enabled='true'"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // setting admin-cluster_mysql_servers_save_to_disk to false + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='0' WHERE variable_name='admin-cluster_mysql_servers_save_to_disk'"); + + // setting admin-cluster_mysql_servers_sync_algorithm to 1 -> fetch mysql_servers_v2 and runtime_mysql_servers + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); + MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + sleep(1); + + const std::vector insert_mysql_servers_values { + std::make_tuple(1, "127.0.0.1", 13306, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, ""), // this server has read_only value 0 (writer) + std::make_tuple(2, "127.0.0.1", 13307, 13, "OFFLINE_SOFT", 2, 1, 500, 300, 1, 200, ""), + std::make_tuple(3, "127.0.0.1", 13308, 14, "OFFLINE_HARD", 2, 1, 500, 300, 1, 200, ""), + std::make_tuple(4, "127.0.0.1", 13309, 15, "SHUNNED", 1, 0, 500, 300, 1, 200, "") + }; + + const std::vector insert_replication_hostgroups_values { + std::make_tuple(0, 1, "read_only") + }; + + // Inserting new records into 'mysql_servers' and 'mysql_replication_hostgroups'. + // Here we are assigning the hostgroup to the reader, and read-only actions will creating a new entry in hostgroup 0. + int result = insert_mysql_servers_records(proxy_admin, insert_mysql_servers_values, insert_replication_hostgroups_values); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Failed to insert records in mysql_servers table."); + res = -1; + goto cleanup; + } + + std::string mysql_servers_sync_algorithm; + int err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); + if (err) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); + res = -1; + goto cleanup; + } + + diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } + + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); + MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + // adding dummy data so replica nodes can sync after algorithm change from 1 to 2. + MYSQL_QUERY(proxy_admin, "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (999,998,'read_only')"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + sleep(1); + + err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); + if (err) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); + res = -1; + goto cleanup; + } + + diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } + + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='3' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); + MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + // deleting dummy data so replica nodes can sync after algorithm change from 2 to 3. + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup=999 AND reader_hostgroup=998 AND check_type='read_only'"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + sleep(2); + + err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); + if (err) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); + res = -1; + goto cleanup; + } + + diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } + + MYSQL_QUERY(proxy_admin, "SET mysql-monitor_enabled='false'"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + sleep(1); + + err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); + if (err) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); + res = -1; + goto cleanup; + } + + diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } + + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); + MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + sleep(1); + + err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); + if (err) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); + res = -1; + goto cleanup; + } + + diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } + + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); + MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + // adding dummy data so replica nodes can sync after algorithm change from 1 to 2. + MYSQL_QUERY(proxy_admin, "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (999,998,'read_only')"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + sleep(1); + + err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); + if (err) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); + res = -1; + goto cleanup; + } + + diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } + + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='3' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); + MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + // deleting dummy data so replica nodes can sync after algorithm change from 2 to 3. + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup=999 AND reader_hostgroup=998 AND check_type='read_only'"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + sleep(1); + + err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); + if (err) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); + res = -1; + goto cleanup; + } + + diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } + } + +cleanup: + // In case of test failing, save the stderr output from the spawned proxysql instance + if (tests_failed() != 0) { + save_proxy_stderr.store(true); + } + + if (r_proxysql_nomonitor_admin) { + int mysql_timeout = 2; + + mysql_options(r_proxysql_nomonitor_admin, MYSQL_OPT_CONNECT_TIMEOUT, &mysql_timeout); + mysql_options(r_proxysql_nomonitor_admin, MYSQL_OPT_READ_TIMEOUT, &mysql_timeout); + mysql_options(r_proxysql_nomonitor_admin, MYSQL_OPT_WRITE_TIMEOUT, &mysql_timeout); + mysql_query(r_proxysql_nomonitor_admin, "PROXYSQL SHUTDOWN"); + mysql_close(r_proxysql_nomonitor_admin); + } + + if (r_proxysql_withmonitor_admin) { + int mysql_timeout = 2; + + mysql_options(r_proxysql_withmonitor_admin, MYSQL_OPT_CONNECT_TIMEOUT, &mysql_timeout); + mysql_options(r_proxysql_withmonitor_admin, MYSQL_OPT_READ_TIMEOUT, &mysql_timeout); + mysql_options(r_proxysql_withmonitor_admin, MYSQL_OPT_WRITE_TIMEOUT, &mysql_timeout); + mysql_query(r_proxysql_withmonitor_admin, "PROXYSQL SHUTDOWN"); + mysql_close(r_proxysql_withmonitor_admin); + } + + proxysql_replica_nomonitor_thd.join(); + proxysql_replica_withmonitor_thd.join(); + + // Recover primary ProxySQL MySQL and ProxySQL servers + diag("RESTORING: Recovering primary configuration..."); + + { + // Recover previous MySQL servers and generate a newer checksum for primary + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS FROM DISK"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + // Insert primary into another Core node config and wait for replication + diag("RESTORING: Inserting primary back into Core nodes"); + bool recovered_servers_st = false; + + std::string insert_query {}; + string_format( + "INSERT INTO proxysql_servers (hostname,port,weight,comment) VALUES ('%s',%d,0,'proxysql')", + insert_query, cl.host, cl.admin_port + ); + + for (const auto& row : core_nodes) { + const std::string host { row[0] }; + const int port = std::stol(row[1]); + MYSQL* c_node_admin = mysql_init(NULL); + + diag("RESTORING: Inserting into node '%s:%d'", host.c_str(), port); + + if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { + const std::string err_msg { + "Connection to core node failed with '" + std::string { mysql_error(c_node_admin) } + "'. Retrying..." + }; + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str()); + mysql_close(c_node_admin); + continue; + } + + int my_rc = mysql_query(c_node_admin, insert_query.c_str()); + if (my_rc == EXIT_SUCCESS) { + mysql_query(c_node_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); + break; + } else { + const std::string err_msg { + "Insert primary into node failed with: '" + std::string { mysql_error(c_node_admin) } + "'" + }; + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str()); + } + } + + // Wait for sync after primary insertion into Core node + std::string check_for_primary {}; + string_format( + "SELECT COUNT(*) FROM proxysql_servers WHERE hostname=='%s' AND port==%d", check_no_primary_query, + cl.host, cl.admin_port + ); + + // Wait for the other nodes to sync ProxySQL servers to include Primary + int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + // Recover the old ProxySQL servers from backup in primary + MYSQL_QUERY(proxy_admin, "DELETE FROM proxysql_servers"); + MYSQL_QUERY(proxy_admin, "INSERT INTO proxysql_servers SELECT * FROM proxysql_servers_sync_test_backup_2687"); + MYSQL_QUERY(proxy_admin, "DROP TABLE proxysql_servers_sync_test_backup_2687"); + MYSQL_QUERY(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); + } + + mysql_close(proxy_admin); + + return exit_status(); +} \ No newline at end of file From e8b24ff290e53a8894737a02e5d5a650325ce4a1 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Thu, 20 Apr 2023 01:32:29 +0500 Subject: [PATCH 09/56] * Added more test cases. * Cleared duplicate code. * Added comments --- .../test_cluster_sync_mysql_servers-t.cpp | 600 +++++++++++------- 1 file changed, 355 insertions(+), 245 deletions(-) diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp index 25d5b09672..e442c992fe 100644 --- a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -299,45 +299,83 @@ int wait_for_node_sync(MYSQL* r_proxy_admin, uint64_t master_checksum, const std }; int check_mysql_servers_sync( - const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_withmonitor_admin, MYSQL* r_proxy_nomonitor_admin, - int cluster_sync_mysql_servers_algorithm + const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_withmonitor_admin, MYSQL* r_proxy_nomonitor_admin ) { std::string print_master_mysql_servers_hostgroups; string_format(t_debug_query, print_master_mysql_servers_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM mysql_servers"); std::string print_master_runtime_mysql_servers_hostgroups; string_format(t_debug_query, print_master_runtime_mysql_servers_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_servers"); - + std::string print_nomonitor_replica_mysql_servers_hostgroups; string_format(t_debug_query, print_nomonitor_replica_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_NOMONITOR_PORT, "SELECT * FROM mysql_servers"); std::string print_nomonitor_replica_runtime_mysql_servers_hostgroups; string_format(t_debug_query, print_nomonitor_replica_runtime_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_NOMONITOR_PORT, "SELECT * FROM runtime_mysql_servers"); + std::string print_nomonitor_replica_disk_mysql_servers_hostgroups; + string_format(t_debug_query, print_nomonitor_replica_disk_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_NOMONITOR_PORT, "SELECT * FROM disk.mysql_servers"); std::string print_withmonitor_replica_mysql_servers_hostgroups; string_format(t_debug_query, print_withmonitor_replica_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_WITHMONITOR_PORT, "SELECT * FROM mysql_servers"); std::string print_withmonitor_replica_runtime_mysql_servers_hostgroups; string_format(t_debug_query, print_withmonitor_replica_runtime_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_WITHMONITOR_PORT, "SELECT * FROM runtime_mysql_servers"); + std::string print_withmonitor_replica_disk_mysql_servers_hostgroups; + string_format(t_debug_query, print_withmonitor_replica_disk_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_WITHMONITOR_PORT, "SELECT * FROM disk.mysql_servers"); - std::cout << "MASTER 'MYSQL SERVERS' TABLE BEFORE SYNC:" << std::endl; - system(print_master_mysql_servers_hostgroups.c_str()); - // Wait till read_only actions have been performed - std::string monitor_read_only_interval; - int g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_interval", monitor_read_only_interval); + std::string variable_val; + + // get mysql_servers sync algorithm value + int g_err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", variable_val); if (g_err) { return EXIT_FAILURE; } - std::string monitor_read_only_timeout {}; - g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_timeout", monitor_read_only_timeout); + const int cluster_sync_mysql_servers_algorithm = atoi(variable_val.c_str()); + + // get monitor enabled variable value + g_err = get_variable_value(proxy_admin, "mysql-monitor_enabled", variable_val); if (g_err) { return EXIT_FAILURE; } - uint64_t wait = std::stol(monitor_read_only_interval) + std::stol(monitor_read_only_timeout); - usleep((wait*1000)*2); + bool monitor_enabled = false; + if (strcasecmp(variable_val.c_str(), "true") == 0 || strcasecmp(variable_val.c_str(), "1") == 0) { + monitor_enabled = true; + } + + // get save-to-disk variable value + g_err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_save_to_disk", variable_val); + if (g_err) { return EXIT_FAILURE; } + bool save_to_disk_value = false; + if (strcasecmp(variable_val.c_str(), "true") == 0 || strcasecmp(variable_val.c_str(), "1") == 0) { + save_to_disk_value = true; + } + + // get read_only interval variable value + g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_interval", variable_val); + if (g_err) { return EXIT_FAILURE; } + const long monitor_read_only_interval = std::stol(variable_val); + + // get read_only timeout variable value + g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_timeout", variable_val); + if (g_err) { return EXIT_FAILURE; } + const long monitor_read_only_timeout = std::stol(variable_val); + + diag("Checking mysql_servers_sync status " + "[admin-cluster_mysql_servers_sync_algorithm:'%d', " + "mysql-monitor_enabled:'%s', " + "admin-cluster_mysql_servers_save_to_disk:'%s'" + "]...", cluster_sync_mysql_servers_algorithm, (monitor_enabled ? "true" : "false"), (save_to_disk_value ? "true" : "false")); + + std::cout << "MASTER 'MYSQL SERVERS' TABLE BEFORE SYNC:" << std::endl; + system(print_master_mysql_servers_hostgroups.c_str()); + std::cout << std::endl; + + // Wait till read_only actions have been performed + uint64_t wait = monitor_read_only_interval + monitor_read_only_timeout; + usleep((wait * 1000) * 2); std::cout << "MASTER 'RUNTIME MYSQL SERVERS' TABLE BEFORE SYNC:" << std::endl; system(print_master_runtime_mysql_servers_hostgroups.c_str()); + std::cout << std::endl; - uint64_t master_mysql_servers_checksum = 0; uint64_t master_runtime_mysql_servers_checksum = 0; - + // fetch master mysql_servers resultset and compute it's hash { MYSQL_QUERY(proxy_admin, "SELECT * FROM mysql_servers"); @@ -354,74 +392,177 @@ int check_mysql_servers_sync( mysql_free_result(mysql_servers_res); } + // This comment is exclusively for this TAP test + // If monitor is enabled, records of runtime_mysql_servers and mysql_servers should not match + if (monitor_enabled == true) { + ok(master_mysql_servers_checksum != master_runtime_mysql_servers_checksum, "'runtime_mysql_servers' and 'mysql_servers' should not match."); + } else { + ok(master_mysql_servers_checksum == master_runtime_mysql_servers_checksum, "'runtime_mysql_servers' and 'mysql_servers' should match."); + } + // + // SYNCH CHECK bool not_synced_query = false; - if (cluster_sync_mysql_servers_algorithm == 1) { - - not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); - std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); - std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); - std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); - std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); - } else if (cluster_sync_mysql_servers_algorithm == 2) { - - not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); - std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); - std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); - std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "runtime_mysql_servers"); - std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); - } else if (cluster_sync_mysql_servers_algorithm == 3) { - - not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); - std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); - std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); - std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); - - not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); - std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; - system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); - ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + if (save_to_disk_value == false) { + + if (cluster_sync_mysql_servers_algorithm == 1) { + // Algo: 1 [Sync mysql_servers_v2 and runtime_mysql_servers] + + // Replica [WITHMONITOR] mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [WITHMONITOR] runtime_mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] runtime_mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + } else if (cluster_sync_mysql_servers_algorithm == 2) { + // Algo: 2 [Sync mysql_servers_v2 only] + + // Replica [WITHMONITOR] mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [WITHMONITOR] runtime_mysql_servers for both the replica and master will be identical. + // Reason: Replica [WITHMONITOR] has monitoring checks enabled and will generate identical runtime_mysql_servers records. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] runtime_mysql_servers will be identical to master mysql_servers. + // Reason: Replica [NOMONITOR] has monitoring checks disabled, so runtime_mysql_servers will be identical to mysql_servers records. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + } else if (cluster_sync_mysql_servers_algorithm == 3) { + // Algo: 3 [If the command line includes the argument "-M", Algorithm 1 will be selected. + // If "-M" is not provided, then Algorithm 2 will be chosen.] + + // Replica [WITHMONITOR] mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] mysql_servers for both the replica and master will be identical. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [WITHMONITOR] runtime_mysql_servers for both the replica and master will be identical. + // Reason: Algorithm 2 will be selected [Sync mysql_servers_v2]. After read_only_action, + // the runtime_mysql_servers for replica becomes identical to master. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] runtime_mysql_servers for both the replica and master will be identical. + // Reason: Algorithm 1 will be selected [Sync mysql_servers_v2 and runtime_mysql_servers] + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "runtime_mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'RUNTIME MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_runtime_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'runtime_mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + } + } else { //save_to_disk_value is true + + // If Algorithm 1 is selected, the runtime_mysql_servers data will be saved to disk. + // If Algorithm 2 is selected, the mysql_servers data will be saved to disk. + // However, for Algorithm 3, the data saved to disk will depend on the algorithm that is chosen based on the -M argument. + + if (cluster_sync_mysql_servers_algorithm == 1) { + + // Replica [WITHMONITOR] disk.mysql_servers and master runtime_mysql_servers will be identical. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_runtime_mysql_servers_checksum, "disk.mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_disk_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'disk.mysql_servers' should be identical to 'runtime_mysql_servers'."); + std::cout << std::endl; + + // Replica [NOMONITOR] disk.mysql_servers and master runtime_mysql_servers will be identical. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "disk.mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_disk_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'disk.mysql_servers' should be identical to 'runtime_mysql_servers'."); + std::cout << std::endl; + + } else if (cluster_sync_mysql_servers_algorithm == 2) { + + // Replica [WITHMONITOR] disk.mysql_servers and master mysql_servers will be identical. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "disk.mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_disk_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'disk.mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] disk.mysql_servers and master mysql_servers will be identical. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_mysql_servers_checksum, "disk.mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_disk_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'disk.mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + } else if (cluster_sync_mysql_servers_algorithm == 3) { + + // Replica [WITHMONITOR] disk.mysql_servers and master mysql_servers will be identical. + not_synced_query = wait_for_node_sync(r_proxy_withmonitor_admin, master_mysql_servers_checksum, "disk.mysql_servers"); + std::cout << "REPLICA [WITHMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_withmonitor_replica_disk_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'disk.disk.mysql_servers' with NULL comments should be synced."); + std::cout << std::endl; + + // Replica [NOMONITOR] disk.mysql_servers and master runtime_mysql_servers will be identical. + not_synced_query = wait_for_node_sync(r_proxy_nomonitor_admin, master_runtime_mysql_servers_checksum, "disk.mysql_servers"); + std::cout << "REPLICA [NOMONITOR] 'MYSQL SERVERS' TABLE AFTER SYNC:" << std::endl; + system(print_nomonitor_replica_disk_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'disk.mysql_servers' with NULL comments should be synced."); + } } + diag("Checking mysql_servers_sync status " + "[admin-cluster_mysql_servers_sync_algorithm:'%d', " + "mysql-monitor_enabled:'%s', " + "admin-cluster_mysql_servers_save_to_disk:'%s'" + "]... Done", cluster_sync_mysql_servers_algorithm, (monitor_enabled ? "true" : "false"), (save_to_disk_value ? "true" : "false")); + return EXIT_SUCCESS; } @@ -574,6 +715,113 @@ int launch_proxysql_replica(const CommandLine& cl, uint32_t r_port, const std::s return EXIT_SUCCESS; } +int get_read_only_value(const std::string& host, uint16_t port, const std::string& username, const std::string& password, + int* read_only_val) { + + // check is mysql server has read_only value 0 + MYSQL* mysqldb = mysql_init(NULL); + + // Initialize connections + if (!mysqldb) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysqldb)); + return EXIT_FAILURE; + } + + // Connnect to local proxysql + if (!mysql_real_connect(mysqldb, host.c_str(), username.c_str(), password.c_str(), NULL, port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysqldb)); + mysql_close(mysqldb); + return EXIT_FAILURE; + } + + const int rc_query = mysql_query(mysqldb,"SELECT @@global.read_only read_only"); + + if (rc_query == 0) { + MYSQL_RES *result = mysql_store_result(mysqldb); + MYSQL_ROW row; + + while ((row = mysql_fetch_row(result))) { + + if (row[0]) { + *read_only_val = static_cast(std::strtoul(row[0], NULL, 10)); + } + } + + mysql_free_result(result); + } + + mysql_close(mysqldb); + + return EXIT_SUCCESS; +} + +std::vector> queries = { + { + "SET mysql-monitor_read_only_interval=200", // setting read_only variables + "SET mysql-monitor_read_only_timeout=100", + "SET mysql-monitor_enabled='true'", // enabling monitor + "LOAD MYSQL VARIABLES TO RUNTIME", + "UPDATE global_variables SET variable_value='false' WHERE variable_name='admin-cluster_mysql_servers_save_to_disk'", // setting admin-cluster_mysql_servers_save_to_disk to false + "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'",// setting admin-cluster_mysql_servers_sync_algorithm to 1 -> fetch mysql_servers_v2 and runtime_mysql_servers + "LOAD ADMIN VARIABLES TO RUNTIME" + }, + { + "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME", + "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (999,998,'read_only')", // adding dummy data so replica nodes can sync after algorithm change from 1 to 2. + "LOAD MYSQL SERVERS TO RUNTIME" + }, + { + "UPDATE global_variables SET variable_value='3' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME", + "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup=999 AND reader_hostgroup=998 AND check_type='read_only'", // deleting dummy data so replica nodes can sync after algorithm change from 2 to 3. + "LOAD MYSQL SERVERS TO RUNTIME" + }, + { + "SET mysql-monitor_enabled='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME" + }, + { + "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME" + }, + { + "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME", + "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (999,998,'read_only')",// adding dummy data so replica nodes can sync after algorithm change from 1 to 2. + "LOAD MYSQL SERVERS TO RUNTIME" + }, + { + "UPDATE global_variables SET variable_value='3' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME", + "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup=999 AND reader_hostgroup=998 AND check_type='read_only'", // deleting dummy data so replica nodes can sync after algorithm change from 2 to 3. + "LOAD MYSQL SERVERS TO RUNTIME" + }, + { + // save to disk + "SET mysql-monitor_enabled='true'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "UPDATE global_variables SET variable_value='true' WHERE variable_name='admin-cluster_mysql_servers_save_to_disk'", // setting admin-cluster_mysql_servers_save_to_disk to true + "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME", + "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (997,996,'read_only')", // adding dummy data so replica nodes can sync after algorithm change from 1 to 2. + "LOAD MYSQL SERVERS TO RUNTIME" + }, + { + "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME", + "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup=997 AND reader_hostgroup=996 AND check_type='read_only'", // deleting dummy data so replica nodes can sync after algorithm change from 2 to 3. + "LOAD MYSQL SERVERS TO RUNTIME" + }, + { + "UPDATE global_variables SET variable_value='3' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + "LOAD ADMIN VARIABLES TO RUNTIME", + "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (997,996,'read_only')",// adding dummy data so replica nodes can sync after algorithm change from 1 to 2. + "LOAD MYSQL SERVERS TO RUNTIME" + } +}; + int main(int, char**) { int res = 0; CommandLine cl; @@ -584,8 +832,12 @@ int main(int, char**) { return EXIT_FAILURE; } - - plan(32); + plan( 1 + 1 // replica instances + + 1 // confirming mysql server 127.0.0.1:13306 is a writer + + (7 * 5) // calling check_mysql_servers_sync 7 times, 5 differnt checks in each call + + (3 * 3) + + 1 + 1 // shutting down replica instances + ); MYSQL* proxy_admin = mysql_init(NULL); @@ -682,19 +934,17 @@ int main(int, char**) { goto cleanup; } - // setting read_only variables and enabling monitor - MYSQL_QUERY(proxy_admin, "SET mysql-monitor_read_only_interval=200"); - MYSQL_QUERY(proxy_admin, "SET mysql-monitor_read_only_timeout=100"); - MYSQL_QUERY(proxy_admin, "SET mysql-monitor_enabled='true'"); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - - // setting admin-cluster_mysql_servers_save_to_disk to false - MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='0' WHERE variable_name='admin-cluster_mysql_servers_save_to_disk'"); - - // setting admin-cluster_mysql_servers_sync_algorithm to 1 -> fetch mysql_servers_v2 and runtime_mysql_servers - MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); - MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); - sleep(1); + int read_only_val = -1; + int result = get_read_only_value("127.0.0.1", 13306, "root", "root", &read_only_val); + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed."); + res = -1; + goto cleanup; + } + + // For thorough testing of synchronization under all possible scenarios, it is necessary for + // the MySQL server at 127.0.0.1:13306 to function as a writer. + ok(read_only_val == 0, "MySQL Server '127.0.0.1:13306' should function as a writer"); const std::vector insert_mysql_servers_values { std::make_tuple(1, "127.0.0.1", 13306, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, ""), // this server has read_only value 0 (writer) @@ -704,12 +954,11 @@ int main(int, char**) { }; const std::vector insert_replication_hostgroups_values { - std::make_tuple(0, 1, "read_only") + std::make_tuple(0, 1, "read_only") // Here we are assigning the hostgroup to the reader, and read-only actions will creating a new entry in hostgroup 0. }; // Inserting new records into 'mysql_servers' and 'mysql_replication_hostgroups'. - // Here we are assigning the hostgroup to the reader, and read-only actions will creating a new entry in hostgroup 0. - int result = insert_mysql_servers_records(proxy_admin, insert_mysql_servers_values, insert_replication_hostgroups_values); + result = insert_mysql_servers_records(proxy_admin, insert_mysql_servers_values, insert_replication_hostgroups_values); if (result != EXIT_SUCCESS) { fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Failed to insert records in mysql_servers table."); @@ -717,158 +966,19 @@ int main(int, char**) { goto cleanup; } - std::string mysql_servers_sync_algorithm; - int err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); - if (err) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); - res = -1; - goto cleanup; - } - - diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); - result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + for (const auto& pre_queries : queries) { - if (result != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; - goto cleanup; - } - - MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); - MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); - - // adding dummy data so replica nodes can sync after algorithm change from 1 to 2. - MYSQL_QUERY(proxy_admin, "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (999,998,'read_only')"); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - sleep(1); - - err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); - if (err) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); - res = -1; - goto cleanup; - } - - diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); - result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); - - if (result != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; - goto cleanup; - } - - MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='3' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); - MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); - - // deleting dummy data so replica nodes can sync after algorithm change from 2 to 3. - MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup=999 AND reader_hostgroup=998 AND check_type='read_only'"); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - sleep(2); - - err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); - if (err) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); - res = -1; - goto cleanup; - } - - diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); - result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); - - if (result != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; - goto cleanup; - } - - MYSQL_QUERY(proxy_admin, "SET mysql-monitor_enabled='false'"); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - sleep(1); - - err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); - if (err) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); - res = -1; - goto cleanup; - } - - diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); - result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); - - if (result != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; - goto cleanup; - } - - MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); - MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); - sleep(1); - - err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); - if (err) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); - res = -1; - goto cleanup; - } - - diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); - result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); - - if (result != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; - goto cleanup; - } - - MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); - MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); - - // adding dummy data so replica nodes can sync after algorithm change from 1 to 2. - MYSQL_QUERY(proxy_admin, "INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (999,998,'read_only')"); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - sleep(1); - - err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); - if (err) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); - res = -1; - goto cleanup; - } - - diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); - result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); - - if (result != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; - goto cleanup; - } - - MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='3' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'"); - MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); - - // deleting dummy data so replica nodes can sync after algorithm change from 2 to 3. - MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup=999 AND reader_hostgroup=998 AND check_type='read_only'"); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - sleep(1); - - err = get_variable_value(proxy_admin, "admin-cluster_mysql_servers_sync_algorithm", mysql_servers_sync_algorithm); - if (err) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching 'admin-cluster_mysql_servers_sync_algorithm' variable value failed."); - res = -1; - goto cleanup; - } - - diag("Checking mysql_servers_sync status... admin-cluster_mysql_servers_sync_algorithm variable value '%s'\n", mysql_servers_sync_algorithm.c_str()); - result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin, std::stol(mysql_servers_sync_algorithm)); + for (const std::string& query : pre_queries) { + MYSQL_QUERY(proxy_admin, query.c_str()); + } + sleep(1); - if (result != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; - goto cleanup; + result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin); + if (result != EXIT_SUCCESS) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); + res = -1; + goto cleanup; + } } } From 5114069018ea76116dfb8e9523e46660e2f6547e Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Thu, 20 Apr 2023 11:27:30 +0500 Subject: [PATCH 10/56] * Fixed test case --- test/tap/tests/test_cluster_sync_mysql_servers-t.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp index e442c992fe..f7e5616fde 100644 --- a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -780,11 +780,9 @@ std::vector> queries = { { "SET mysql-monitor_enabled='false'", "LOAD MYSQL VARIABLES TO RUNTIME", - "LOAD MYSQL SERVERS TO RUNTIME" - }, - { "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", "LOAD ADMIN VARIABLES TO RUNTIME" + "LOAD MYSQL SERVERS TO RUNTIME" }, { "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", @@ -834,7 +832,7 @@ int main(int, char**) { plan( 1 + 1 // replica instances + 1 // confirming mysql server 127.0.0.1:13306 is a writer - + (7 * 5) // calling check_mysql_servers_sync 7 times, 5 differnt checks in each call + + (6 * 5) // calling check_mysql_servers_sync 7 times, 5 differnt checks in each call + (3 * 3) + 1 + 1 // shutting down replica instances ); From f35003802ed709edf05964e67e0ccc525416a50d Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Thu, 20 Apr 2023 15:47:46 +0500 Subject: [PATCH 11/56] * The cluster template file now includes "cluster_sync_interfaces=false." * In the event of any failure, it's crucial to call the cleanup function to prevent any orphaned replica instances from remaining active. --- .../test_cluster_sync-t.cnf | 1 + .../tests/test_cluster_sync_mysql_servers-t.cpp | 14 ++++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf b/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf index 8a51f47dc9..51b163f34e 100644 --- a/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf +++ b/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf @@ -17,6 +17,7 @@ admin_variables= cluster_mysql_users_diffs_before_sync=3 cluster_admin_variables_diffs_before_sync=3 cluster_proxysql_servers_diffs_before_sync=3 + cluster_sync_interfaces=false } mysql_variables= diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp index f7e5616fde..3ae8f11285 100644 --- a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -781,7 +781,7 @@ std::vector> queries = { "SET mysql-monitor_enabled='false'", "LOAD MYSQL VARIABLES TO RUNTIME", "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", - "LOAD ADMIN VARIABLES TO RUNTIME" + "LOAD ADMIN VARIABLES TO RUNTIME", "LOAD MYSQL SERVERS TO RUNTIME" }, { @@ -821,7 +821,7 @@ std::vector> queries = { }; int main(int, char**) { - int res = 0; + CommandLine cl; std::atomic save_proxy_stderr(false); @@ -910,7 +910,6 @@ int main(int, char**) { if (r_proxysql_nomonitor_admin == nullptr) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxysql_nomonitor_admin)); - res = -1; goto cleanup; } @@ -928,7 +927,6 @@ int main(int, char**) { if (r_proxysql_withmonitor_admin == nullptr) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxysql_withmonitor_admin)); - res = -1; goto cleanup; } @@ -936,7 +934,6 @@ int main(int, char**) { int result = get_read_only_value("127.0.0.1", 13306, "root", "root", &read_only_val); if (result != EXIT_SUCCESS) { fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed."); - res = -1; goto cleanup; } @@ -960,21 +957,22 @@ int main(int, char**) { if (result != EXIT_SUCCESS) { fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Failed to insert records in mysql_servers table."); - res = -1; goto cleanup; } for (const auto& pre_queries : queries) { for (const std::string& query : pre_queries) { - MYSQL_QUERY(proxy_admin, query.c_str()); + if (mysql_query(proxy_admin, query.c_str())) { + fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + goto cleanup; + } } sleep(1); result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin); if (result != EXIT_SUCCESS) { fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Checking mysql servers sync records failed."); - res = -1; goto cleanup; } } From 082a59d4c9eb4a44f6cdb9a46306ebbdd0824296 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Fri, 21 Apr 2023 00:58:18 +0500 Subject: [PATCH 12/56] Fixed test case. --- .../test_cluster_sync_mysql_servers-t.cpp | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp index 3ae8f11285..f345b287d6 100644 --- a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -585,9 +585,7 @@ int update_proxysql_servers(const CommandLine& cl, MYSQL* admin) { } int setup_config_file(const CommandLine& cl, uint32_t r_port, const std::string& config_filename) { - const std::string& workdir = std::string(cl.workdir); - const std::string& t_fmt_config_file = workdir + "test_cluster_sync_config/test_cluster_sync-t.cnf"; const std::string& fmt_config_file = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/test_cluster_sync.cnf"; const std::string& datadir_path = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename; @@ -677,22 +675,18 @@ int setup_config_file(const CommandLine& cl, uint32_t r_port, const std::string& int launch_proxysql_replica(const CommandLine& cl, uint32_t r_port, const std::string config_filename, bool monitor_enabled, const std::atomic& save_proxy_stderr) { - // Setup the config file using the env variables in 'CommandLine' - if (setup_config_file(cl, r_port, config_filename)) { - return EXIT_FAILURE; - } - const std::string& workdir = std::string(cl.workdir); - const std::string& replica_stderr = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/cluster_sync_node_stderr.txt"; const std::string& proxysql_db = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/proxysql.db"; const std::string& stats_db = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/proxysql_stats.db"; const std::string& fmt_config_file = workdir + "test_cluster_sync_config/test_cluster_sync_" + config_filename + "/test_cluster_sync.cnf"; - std::string proxy_stdout {}; - std::string proxy_stderr {}; - const std::string& proxy_binary_path = workdir + "../../../src/proxysql"; + // Setup the config file using the env variables in 'CommandLine' + if (setup_config_file(cl, r_port, config_filename)) { + return EXIT_FAILURE; + } + const std::string& proxy_binary_path = workdir + "../../../src/proxysql"; const std::string& proxy_command = proxy_binary_path + " -f " + (monitor_enabled == false ? "-M" : "") + " -c " + fmt_config_file + " > " + replica_stderr + " 2>&1"; diag("Launching replica ProxySQL [%s] via 'system' with command : `%s`", config_filename.c_str(), proxy_command.c_str()); @@ -761,7 +755,6 @@ std::vector> queries = { "SET mysql-monitor_read_only_timeout=100", "SET mysql-monitor_enabled='true'", // enabling monitor "LOAD MYSQL VARIABLES TO RUNTIME", - "UPDATE global_variables SET variable_value='false' WHERE variable_name='admin-cluster_mysql_servers_save_to_disk'", // setting admin-cluster_mysql_servers_save_to_disk to false "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'",// setting admin-cluster_mysql_servers_sync_algorithm to 1 -> fetch mysql_servers_v2 and runtime_mysql_servers "LOAD ADMIN VARIABLES TO RUNTIME" }, @@ -782,7 +775,7 @@ std::vector> queries = { "LOAD MYSQL VARIABLES TO RUNTIME", "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", "LOAD ADMIN VARIABLES TO RUNTIME", - "LOAD MYSQL SERVERS TO RUNTIME" + "LOAD MYSQL SERVERS TO RUNTIME" // to regenerate runtime_mysql_servers }, { "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", @@ -885,7 +878,14 @@ int main(int, char**) { MYSQL_QUERY(proxy_admin, update_proxysql_servers.c_str()); MYSQL_QUERY(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); - + // disable admin-cluster_mysql_servers_save_to_disk before executing replicas + MYSQL_QUERY(proxy_admin, "UPDATE global_variables SET variable_value='false' WHERE variable_name='admin-cluster_mysql_servers_save_to_disk'"); // setting admin-cluster_mysql_servers_save_to_disk to false + MYSQL_QUERY(proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + // cleaning old records + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_servers"); + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups"); + // Launch proxysql with cluster config and monitor feature disabled std::thread proxysql_replica_nomonitor_thd(launch_proxysql_replica, std::ref(cl), R_NOMONITOR_PORT, "nomonitor", false, std::ref(save_proxy_stderr)); @@ -963,12 +963,10 @@ int main(int, char**) { for (const auto& pre_queries : queries) { for (const std::string& query : pre_queries) { - if (mysql_query(proxy_admin, query.c_str())) { - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, mysql_error(proxy_admin)); - goto cleanup; - } + MYSQL_QUERY__(proxy_admin, query.c_str()); + usleep(100000); } - sleep(1); + sleep(2); result = check_mysql_servers_sync(cl, proxy_admin, r_proxysql_withmonitor_admin, r_proxysql_nomonitor_admin); if (result != EXIT_SUCCESS) { @@ -1074,4 +1072,4 @@ int main(int, char**) { mysql_close(proxy_admin); return exit_status(); -} \ No newline at end of file +} From 9ad8c38a7003cc318fbccf8736b593f0221656b8 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Wed, 3 May 2023 12:02:22 +0500 Subject: [PATCH 13/56] Fixed tap test --- lib/ProxySQL_Cluster.cpp | 2 +- test/tap/tests/test_cluster_sync-t.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index dd5a766f12..ff4471618b 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -3205,7 +3205,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_v2(char** host, uint if (v->version > 1) { if (v->epoch > epoch) { max_epoch = v->epoch; - if (v->diff_check > diff_ms) { + if (v->diff_check >= diff_ms) { epoch = v->epoch; version = v->version; if (mysql_servers_v2_checksum) { diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index e5897061ec..e8b30d5dfa 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -502,7 +502,7 @@ int update_proxysql_servers(const CommandLine& cl, MYSQL* admin) { const vector module_sync_payloads { { update_mysql_servers, - "mysql_servers", + "mysql_servers_v2", "admin-cluster_mysql_servers_diffs_before_sync", }, { @@ -2156,6 +2156,7 @@ int main(int, char**) { std::make_tuple("admin-cluster_mysql_variables_save_to_disk" , "true" ), std::make_tuple("admin-cluster_proxysql_servers_diffs_before_sync" , "4" ), std::make_tuple("admin-cluster_proxysql_servers_save_to_disk" , "true" ), + std::make_tuple("admin-cluster_mysql_servers_sync_algorithm" , "1" ), // std::make_tuple("admin-cluster_username" , "" ), Known issue, can't clear // std::make_tuple("admin-cluster_password" , "" ), Known issue, can't clear // std::make_tuple("admin-debug" , "false" ), Should not be synced From 1decc6a993b5aa951c9e038c973b9d1ae4741d74 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Thu, 4 May 2023 13:37:52 +0500 Subject: [PATCH 14/56] Added delay to avoid race condition in tap test --- test/tap/tests/test_cluster_sync_mysql_servers-t.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp index f345b287d6..8d258710fe 100644 --- a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -773,9 +773,9 @@ std::vector> queries = { { "SET mysql-monitor_enabled='false'", "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", // to regenerate runtime_mysql_servers "UPDATE global_variables SET variable_value='1' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", - "LOAD ADMIN VARIABLES TO RUNTIME", - "LOAD MYSQL SERVERS TO RUNTIME" // to regenerate runtime_mysql_servers + "LOAD ADMIN VARIABLES TO RUNTIME" }, { "UPDATE global_variables SET variable_value='2' WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", @@ -885,6 +885,7 @@ int main(int, char**) { // cleaning old records MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_servers"); MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); // Launch proxysql with cluster config and monitor feature disabled std::thread proxysql_replica_nomonitor_thd(launch_proxysql_replica, std::ref(cl), R_NOMONITOR_PORT, "nomonitor", false, std::ref(save_proxy_stderr)); @@ -964,7 +965,7 @@ int main(int, char**) { for (const std::string& query : pre_queries) { MYSQL_QUERY__(proxy_admin, query.c_str()); - usleep(100000); + usleep(1000000); } sleep(2); From 129737caafa2bc42c1c1ad2e20d57404047ada77 Mon Sep 17 00:00:00 2001 From: Miro Stauder Date: Thu, 4 May 2023 11:13:17 +0000 Subject: [PATCH 15/56] fix old compiler issue --- lib/MySQL_HostGroups_Manager.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c7e0d5ab15..f7bfc75eea 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1879,7 +1879,9 @@ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { if (itr == hostgroup_server_mapping.end()) { std::unique_ptr server_mapping(new HostGroup_Server_Mapping(this)); fetched_server_mapping = server_mapping.get(); - hostgroup_server_mapping.insert({ server_id, std::move(server_mapping) }); + hostgroup_server_mapping.insert( std::pair> { + server_id, std::move(server_mapping) + } ); } else { fetched_server_mapping = itr->second.get(); } From 9a5a9ac66e35861d5833190996a0dc15659dfec0 Mon Sep 17 00:00:00 2001 From: Miro Stauder Date: Thu, 4 May 2023 11:13:46 +0000 Subject: [PATCH 16/56] fix clang issue --- lib/ProxySQL_Cluster.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index ff4471618b..996655ceee 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -930,8 +930,8 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { proxy_info("Cluster: Fetch mysql_servers_v2:'YES', mysql_servers:'%s' from peer %s:%d\n", (fetch_runtime ? "YES" : "NO"), hostname, port); - GloProxyCluster->pull_mysql_servers_v2_from_peer({ v->checksum, v->epoch }, - { runtime_mysql_server_checksum->checksum, runtime_mysql_server_checksum->epoch }, fetch_runtime); + GloProxyCluster->pull_mysql_servers_v2_from_peer({ v->checksum, static_cast(v->epoch) }, + { runtime_mysql_server_checksum->checksum, static_cast(runtime_mysql_server_checksum->epoch) }, fetch_runtime); runtime_mysql_servers_already_loaded = fetch_runtime; } @@ -967,7 +967,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (v->diff_check >= diff_ms) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_runtime_mysql_servers_from_peer({ v->checksum, v->epoch }); + GloProxyCluster->pull_runtime_mysql_servers_from_peer({ v->checksum, static_cast(v->epoch) }); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms * 10)) == 0)) { From 09b2a7d9d2743bb05670ef8102e20abd6de357ce Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Mon, 8 May 2023 15:55:47 +0500 Subject: [PATCH 17/56] Avoid incrementing mysql_servers and mysql_servers_v2 versions when change is triggered from monitoring module --- include/MySQL_HostGroups_Manager.h | 2 +- lib/MySQL_HostGroups_Manager.cpp | 13 +++++++++---- lib/ProxySQL_Admin.cpp | 2 +- lib/ProxySQL_Cluster.cpp | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 6936b4f054..84495490fc 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -730,7 +730,7 @@ class MySQL_HostGroups_Manager { //void update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, SQLite3_result* mysql_servers_v2 = nullptr, const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {}, - bool only_commit_runtime_mysql_servers = false); + bool only_commit_runtime_mysql_servers = false, bool update_version = false); void commit_update_checksums_from_tables(); void CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c7e0d5ab15..d0939ff2b3 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1907,7 +1907,7 @@ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { bool MySQL_HostGroups_Manager::commit( SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, SQLite3_result* mysql_servers_v2, const mysql_servers_v2_checksum_t& peer_mysql_server_v2, - bool only_commit_runtime_mysql_servers + bool only_commit_runtime_mysql_servers, bool update_version ) { // if only_commit_runtime_mysql_servers is true, mysql_servers_v2 resultset will not be entertained and will cause memory leak. if (only_commit_runtime_mysql_servers) { @@ -2182,7 +2182,10 @@ bool MySQL_HostGroups_Manager::commit( sprintf(buf, "0x%0X%0X", d32[0], d32[1]); pthread_mutex_lock(&GloVars.checksum_mutex); GloVars.checksums_values.mysql_servers.set_checksum(buf); - GloVars.checksums_values.mysql_servers.version++; + + if (update_version) + GloVars.checksums_values.mysql_servers.version++; + //struct timespec ts; //clock_gettime(CLOCK_REALTIME, &ts); if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && @@ -2196,7 +2199,9 @@ bool MySQL_HostGroups_Manager::commit( memcpy(&d32, &mysql_servers_v2_checksum, sizeof(mysql_servers_v2_checksum)); sprintf(buf, "0x%0X%0X", d32[0], d32[1]); GloVars.checksums_values.mysql_servers_v2.set_checksum(buf); - GloVars.checksums_values.mysql_servers_v2.version++; + + if (update_version) + GloVars.checksums_values.mysql_servers_v2.version++; if (peer_mysql_server_v2.epoch != 0 && peer_mysql_server_v2.checksum.empty() == false && GloVars.checksums_values.mysql_servers_v2.checksum == peer_mysql_server_v2.checksum) { @@ -5128,7 +5133,7 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listcommit(runtime_mysql_servers, peer_runtime_mysql_server, incoming_mysql_servers_v2, peer_mysql_server_v2); + MyHGM->commit(runtime_mysql_servers, peer_runtime_mysql_server, incoming_mysql_servers_v2, peer_mysql_server_v2, false, true); // quering runtime table will update and return latest records, so this is not needed. // GloAdmin->save_mysql_servers_runtime_to_database(true); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index ff4471618b..8fbc511c94 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1871,7 +1871,7 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading runtime_mysql_servers from peer %s:%d into mysql_servers_incoming", hostname, port); MyHGM->servers_add(runtime_mysql_servers_resultset.get()); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Updating runtime_mysql_servers from peer %s:%d", hostname, port); - MyHGM->commit(runtime_mysql_servers_resultset.release(), peer_runtime_mysql_server, nullptr, {}, true); + MyHGM->commit(runtime_mysql_servers_resultset.release(), peer_runtime_mysql_server, nullptr, {}, true, true); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving Runtime MySQL Servers to Database\n"); From b2119ba20312291bf4f25b00957f040de02370ca Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Mon, 8 May 2023 16:42:52 +0500 Subject: [PATCH 18/56] Fixed typo --- include/proxysql_admin.h | 2 +- lib/MySQL_HostGroups_Manager.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index a6b87fa9c9..09328df002 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -130,7 +130,7 @@ struct incoming_servers_t { incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); }; -// Seperate structs for runtime mysql server and mysql server v2 to avoid human error +// Separate structs for runtime mysql server and mysql server v2 to avoid human error struct runtime_mysql_servers_checksum_t { std::string checksum; time_t epoch; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f95361bd86..b4993d3894 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1890,7 +1890,7 @@ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { } HostGroup_Server_Mapping::Node node; - node.server_status = static_cast(atoi(r->fields[3])); + //node.server_status = static_cast(atoi(r->fields[3])); node.reader_hostgroup_id = atoi(r->fields[4]); node.writer_hostgroup_id = atoi(r->fields[5]); node.srv = reinterpret_cast(atoll(r->fields[6])); From ef3d6bd5a7c7c518d6c43bf8f750967642959348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 8 May 2023 09:54:49 +0200 Subject: [PATCH 19/56] Fix DEBUG 'conn_unregister' flow for GR monitoring with async_handlers --- lib/MySQL_Monitor.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a0b67343c6..829b365eed 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -212,6 +212,9 @@ class MySQL_Monitor_Connection_Pool { __conn_register_label: for (unsigned int i=0; ilen; i++) { MYSQL *my1 = (MYSQL *)conns->index(i); + // 'my1' can be NULL due to connection cleanup + if (my1 == nullptr) continue; + assert(my!=my1); //assert(my->net.fd!=my1->net.fd); // FIXME: we changed this with the next section of code if (my->net.fd == my1->net.fd) { @@ -232,6 +235,7 @@ class MySQL_Monitor_Connection_Pool { #endif // DEBUG return; }; + // TODO: Document 'unregister' flow related async fetching and 'task_handlers'. void conn_unregister(MySQL_Monitor_State_Data *mmsd) { #ifdef DEBUG std::lock_guard lock(mutex); @@ -322,6 +326,9 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port, if (my) { for (unsigned int j=0; jlen; j++) { MYSQL *my1 = (MYSQL *)conns->index(j); + // 'my1' can be NULL due to connection cleanup + if (!my1) continue; + assert(my!=my1); assert(my->net.fd!=my1->net.fd); } @@ -3695,12 +3702,16 @@ void gr_handle_actions_over_unresp_srvs(const vector& hosts_defs, * before placing the connection back into the 'ConnectionPool', on failure, we discard the connection. * @param mmsd The mmsd wrapper holding all information for returning the connection. */ -void handle_mmsd_mysql_conn(MySQL_Monitor_State_Data* mmsd, bool unregister_conn_on_failure) { +void handle_mmsd_mysql_conn(MySQL_Monitor_State_Data* mmsd) { if (mmsd == nullptr) return; if (mmsd->mysql) { if (mmsd->interr || mmsd->mysql_error_msg) { - if (unregister_conn_on_failure) { + // If 'MySQL_Monitor_State_Data' reaches the end of a task_handler without 'TASK_RESULT_UNKNOWN': + // 1. Connection failed to be created, 'task_result' should be 'TASK_RESULT_UNKNOWN'. No + // unregister needed. + // 2. Fetching operation failed, the async fetching handler already handled the 'unregister'. + if (mmsd->get_task_result() == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { GloMyMon->My_Conn_Pool->conn_unregister(mmsd); } mysql_close(mmsd->mysql); @@ -3717,7 +3728,7 @@ void handle_mmsd_mysql_conn(MySQL_Monitor_State_Data* mmsd, bool unregister_conn MyHGM->p_update_mysql_error_counter( p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, mysql_errno(mmsd->mysql) ); - if (unregister_conn_on_failure) { + if (mmsd->get_task_result() == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { GloMyMon->My_Conn_Pool->conn_unregister(mmsd); } mysql_close(mmsd->mysql); @@ -3766,7 +3777,7 @@ void gr_report_fetching_errs(MySQL_Monitor_State_Data* mmsd) { * @param mmsd The server 'MySQL_Monitor_State_Data' after the fetching is completed. It should either * hold a valid 'MYSQL_RES' or an error. */ -void async_gr_mon_actions_handler(MySQL_Monitor_State_Data* mmsd, bool unregister_conn_on_failure) { +void async_gr_mon_actions_handler(MySQL_Monitor_State_Data* mmsd) { // We base 'start_time' on the conn init for 'MySQL_Monitor_State_Data'. If a conn creation was // required, we take into account this time into account, otherwise we asume that 'start_time=t1'. uint64_t start_time = 0; @@ -3789,7 +3800,7 @@ void async_gr_mon_actions_handler(MySQL_Monitor_State_Data* mmsd, bool unregiste } // Handle 'mmsd' MySQL conn return to 'ConnectionPool' - handle_mmsd_mysql_conn(mmsd, unregister_conn_on_failure); + handle_mmsd_mysql_conn(mmsd); } /** @@ -3905,7 +3916,7 @@ void* monitor_GR_thread_HG(void *arg) { // Handle 'mmsds' that failed to optain conns for (const unique_ptr& mmsd : fail_mmsds) { - async_gr_mon_actions_handler(mmsd.get(), true); + async_gr_mon_actions_handler(mmsd.get()); } // Update 't1' for subsequent fetch operations and reset errors @@ -7554,7 +7565,7 @@ bool MySQL_Monitor::monitor_group_replication_process_ready_tasks_2( for (MySQL_Monitor_State_Data* mmsd : mmsds) { const MySQL_Monitor_State_Data_Task_Result task_result = mmsd->get_task_result(); assert(task_result != MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_PENDING); - async_gr_mon_actions_handler(mmsd, false); + async_gr_mon_actions_handler(mmsd); } return true; From af80944cfe2bceba556a1c20d66e0b757a35d8bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 8 May 2023 09:56:44 +0200 Subject: [PATCH 20/56] Add support for Group Replication (GR) autodiscovery - Add support for autodiscovery for GR for MySQL8. - Allow to configure autodiscovered servers defaults via 'mysql_hostgroup_attributes'. - Add support for GR autodiscovery for Cluster Simulator. - Improve server 'viable_candidate' detection for MySQL 8, 'RECOVERING' state is now also considered. --- include/MySQL_HostGroups_Manager.h | 36 +++++++++ include/MySQL_Monitor.hpp | 16 ++++ include/SQLite3_Server.h | 3 +- include/proxysql_utils.h | 7 ++ lib/MySQL_HostGroups_Manager.cpp | 70 ++++++++++++++++- lib/MySQL_Monitor.cpp | 120 ++++++++++++++++++++++++----- lib/proxysql_utils.cpp | 12 +++ src/SQLite3_Server.cpp | 22 +++--- 8 files changed, 252 insertions(+), 34 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 675cf090e2..92a3add129 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -585,6 +585,17 @@ class MySQL_HostGroups_Manager { SQLite3_result *incoming_replication_hostgroups; void generate_mysql_group_replication_hostgroups_table(); + /** + * @brief Regenerates the resultset used by 'MySQL_Monitor' containing the servers to be monitored. + * @details This function is required to be called after any action that results in the addition of a new + * server that 'MySQL_Monitor' should be aware of for 'group_replication', i.e. a server added to the + * hostgroups present in any entry of 'mysql_group_replication_hostgroups'. E.g: + * - Inside 'generate_mysql_group_replication_hostgroups_table'. + * - Autodiscovery. + * + * NOTE: This is a common pattern for all the clusters monitoring. + */ + void generate_mysql_group_replication_hostgroups_monitor_resultset(); SQLite3_result *incoming_group_replication_hostgroups; pthread_mutex_t Group_Replication_Info_mutex; @@ -812,6 +823,31 @@ class MySQL_HostGroups_Manager { void update_group_replication_set_offline(char *_hostname, int _port, int _writer_hostgroup, char *error); void update_group_replication_set_read_only(char *_hostname, int _port, int _writer_hostgroup, char *error); void update_group_replication_set_writer(char *_hostname, int _port, int _writer_hostgroup); + /** + * @brief Tries to add a new server found during GR autodiscovery to the supplied hostgroup. + * @details For adding the new server, several actions are performed: + * 1. Lookup the target server in the corresponding MyHGC for the supplied hostgroup. + * 2. If server is found, and it's status isn't 'OFFLINE_HARD' do nothing. Otherwise: + * - If server is found as 'OFFLINE_HARD', re-enable the server, log the action. + * - If server isn't found, create it in the corresponding reader hostgroup of the supplied writer + * hostgroup, setting all 'servers_defaults' params as '-1', log the action.pasalo + * - After any of the two previous actions, always regenerate servers data structures. + * + * NOTE: Server data structures regeneration requires: + * 1. Purging the 'mysql_servers_table' (Lazy removal of 'OFFLINE_HARD' servers.) + * 2. Regenerate the actual 'myhgm::mysql_servers' table from memory structures. + * 3. Update the 'mysql_servers' resultset used for monitoring. This resultset is used for general + * monitoring actions like 'ping', 'connect'. + * 4. Regenerate the specific resultset for 'Group Replication' monitoring. This resultset is the way to + * communicate back to the main monitoring thread that servers config has changed, and a new thread + * shall be created with the new servers config. This same principle is used for Aurora. + * + * @param _host Server address. + * @param _port Server port. + * @param _wr_hg Writer hostgroup of the cluster being monitored. Autodiscovered servers are always added + * to the reader hostgroup by default, later monitoring actions will re-position the server is required. + */ + void update_group_replication_add_autodiscovered(const std::string& _host, int _port, int _wr_hg); void converge_group_replication_config(int _writer_hostgroup); /** * @brief Set the supplied server as SHUNNED, this function shall be called diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index ffe186371c..04b08b33f8 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -200,6 +200,17 @@ enum class MySQL_Monitor_State_Data_Task_Result { TASK_RESULT_PENDING }; +/** + * @brief Holds the info from a GR server definition. + */ +struct gr_host_def_t { + string host; + int port; + int use_ssl; + bool writer_is_also_reader; + int max_transactions_behind; + int max_transactions_behind_count; +}; class MySQL_Monitor_State_Data { public: @@ -237,6 +248,11 @@ class MySQL_Monitor_State_Data { * @details Currently only used by 'group_replication'. */ uint64_t init_time = 0; + /** + * @brief Used by GroupReplication to determine if servers reported by cluster 'members' are already monitored. + * @details This way we avoid non-needed locking on 'MySQL_HostGroups_Manager' for server search. + */ + const std::vector* cur_monitored_gr_srvs = nullptr; MySQL_Monitor_State_Data(MySQL_Monitor_State_Data_Task_Type task_type, char* h, int p, bool _use_ssl = 0, int g = 0); ~MySQL_Monitor_State_Data(); diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index ce33a6b6ec..818c6971fc 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -5,6 +5,7 @@ #include "proxysql.h" #include "cpp.h" #include +#include class SQLite3_Session { public: @@ -14,7 +15,7 @@ class SQLite3_Session { }; #ifdef TEST_GROUPREP -using group_rep_status = std::tuple; +using group_rep_status = std::tuple; #endif class SQLite3_Server { diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 2d75c4693f..76a1c2e302 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -203,6 +203,13 @@ uint64_t get_timestamp_us(); */ std::string replace_str(const std::string& str, const std::string& match, const std::string& repl); +/** + * @brief Split a string into a vector of strings with the provided 'char' delimiter. + * @param s String to be split. + * @param delimiter Delimiter to be used. + * @return Vector with the string splits. Empty if none is found. + */ +std::vector split_str(const std::string& s, char delimiter); std::string generate_multi_rows_query(int rows, int params); #endif diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f85f88fcbc..929203536f 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2427,6 +2427,11 @@ void MySQL_HostGroups_Manager::generate_mysql_group_replication_hostgroups_table // it is now time to build a new structure in Monitor + generate_mysql_group_replication_hostgroups_monitor_resultset(); + pthread_mutex_unlock(&Group_Replication_Info_mutex); +} + +void MySQL_HostGroups_Manager::generate_mysql_group_replication_hostgroups_monitor_resultset() { pthread_mutex_lock(&GloMyMon->group_replication_mutex); { char *error=NULL; @@ -2445,8 +2450,6 @@ void MySQL_HostGroups_Manager::generate_mysql_group_replication_hostgroups_table } } pthread_mutex_unlock(&GloMyMon->group_replication_mutex); - - pthread_mutex_unlock(&Group_Replication_Info_mutex); } void MySQL_HostGroups_Manager::generate_mysql_galera_hostgroups_table() { @@ -5645,6 +5648,69 @@ void MySQL_HostGroups_Manager::converge_group_replication_config(int _writer_hos pthread_mutex_unlock(&Group_Replication_Info_mutex); } +void MySQL_HostGroups_Manager::update_group_replication_add_autodiscovered( + const string& _host, int _port, int _wr_hg +) { + pthread_mutex_lock(&Group_Replication_Info_mutex); + const auto gr_info_map_it = this->Group_Replication_Info_Map.find(_wr_hg); + int32_t reader_hg = -1; + + if (gr_info_map_it == Group_Replication_Info_Map.end()) { + assert(0); + } else { + reader_hg = gr_info_map_it->second->reader_hostgroup; + } + pthread_mutex_unlock(&Group_Replication_Info_mutex); + + wrlock(); + + MyHGC *myhgc = MyHGC_lookup(reader_hg); + bool srv_found = false; + bool srv_found_offline = false; + + for (uint32_t j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC* mysrvc = static_cast(myhgc->mysrvs->servers->index(j)); + + // TODO: Motivation for this logic needs to be better described + if (strcmp(mysrvc->address,_host.c_str())==0 && mysrvc->port==_port) { + srv_found = true; + if (mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + proxy_info( + "Found healthy previously discovered GR node %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:" + " hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + _host.c_str(), _port, reader_hg, mysrvc->weight, mysrvc->max_connections, mysrvc->use_ssl + ); + mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; + srv_found_offline = true; + } + } + } + + if (srv_found == false) { + MySrvC* mysrvc = new MySrvC( + const_cast(_host.c_str()), _port, 0, -1, MYSQL_SERVER_STATUS_ONLINE, 0, -1, 0, -1, 0, const_cast("") + ); + add(mysrvc, reader_hg); + proxy_info( + "Adding new discovered GR node %s:%d with: hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + _host.c_str(), _port, reader_hg, mysrvc->weight, mysrvc->max_connections, mysrvc->use_ssl + ); + } + + if (srv_found == false || srv_found_offline) { + purge_mysql_servers_table(); + + mydb->execute("DELETE FROM mysql_servers"); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + + generate_mysql_servers_table(); + update_table_mysql_servers_for_monitor(false); + generate_mysql_group_replication_hostgroups_monitor_resultset(); + } + + wrunlock(); +} + Galera_Info::Galera_Info(int w, int b, int r, int o, int mw, int mtb, bool _a, int _w, char *c) { comment=NULL; if (c) { diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 829b365eed..efaa9a1dab 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -495,19 +495,23 @@ void MySQL_Monitor_Connection_Pool::put_connection(char *hostname, int port, MYS /** * @brief MySQL 8 status query for Group Replication members. * @details Since 'MySQL 8' we rely on 'COUNT_TRANSACTIONS_REMOTE_IN_APPLIER_QUEUE', deprecating the previously - * required 'sys.gr_member_routing_candidate_status' view. + * required 'sys.gr_member_routing_candidate_status' view. Another additions: + * - A new field 'members' has been added to the query, containing the current cluster members as seen by the + * queried node. This field is used for auto discovery. + * - Server state 'RECOVERING' is now also considered when detecting if a member is a 'viable' candidate. */ const char MYSQL_8_GR_QUERY[] { "SELECT (SELECT IF (" "MEMBER_STATE='ONLINE' AND (" - "(SELECT COUNT(*) FROM performance_schema.replication_group_members WHERE MEMBER_STATE != 'ONLINE') >=" + "(SELECT COUNT(*) FROM performance_schema.replication_group_members WHERE MEMBER_STATE NOT IN ('ONLINE', 'RECOVERING')) >=" " ((SELECT COUNT(*) FROM performance_schema.replication_group_members)/2) = 0)" ", 'YES', 'NO')) AS viable_candidate," " (SELECT IF (@@read_only, 'YES', 'NO')) as read_only," - " COUNT_TRANSACTIONS_REMOTE_IN_APPLIER_QUEUE AS transactions_behind " + " COUNT_TRANSACTIONS_REMOTE_IN_APPLIER_QUEUE AS transactions_behind, " + " (SELECT GROUP_CONCAT(CONCAT(member_host, \":\", member_port)) FROM performance_schema.replication_group_members) AS members " "FROM " "performance_schema.replication_group_members " - "JOIN performance_schema.replication_group_member_stats rgms USING(member_id)" + "JOIN performance_schema.replication_group_member_stats rgms USING(member_id) " "WHERE rgms.MEMBER_ID=@@SERVER_UUID" }; @@ -616,7 +620,7 @@ void MySQL_Monitor_State_Data::init_async() { async_state_machine_ = ASYNC_QUERY_START; #ifdef TEST_GROUPREP { - query_ = "SELECT viable_candidate,read_only,transactions_behind FROM GR_MEMBER_ROUTING_CANDIDATE_STATUS "; + query_ = "SELECT viable_candidate,read_only,transactions_behind,members FROM GR_MEMBER_ROUTING_CANDIDATE_STATUS "; query_ += std::string(hostname) + ":" + std::to_string(port); } #else @@ -1867,7 +1871,7 @@ void * monitor_group_replication_thread(void *arg) { mmsd->interr=0; // reset the value #ifdef TEST_GROUPREP { - std::string s { "SELECT viable_candidate,read_only,transactions_behind FROM GR_MEMBER_ROUTING_CANDIDATE_STATUS" }; + std::string s { "SELECT viable_candidate,read_only,transactions_behind,members FROM GR_MEMBER_ROUTING_CANDIDATE_STATUS" }; s += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,s.c_str()); } @@ -3373,18 +3377,6 @@ set extract_writer_hgs(SQLite3_result* Group_Replication_Hosts_results return writer_hgs; } -/** - * @brief Holds the info from a GR server definition. - */ -typedef struct _gr_host_def_t { - string host; - int port; - int use_ssl; - bool writer_is_also_reader; - int max_transactions_behind; - int max_transactions_behind_count; -} gr_host_def_t; - /** * @brief Extracts a 'MySQL_Monitor_State_Data' from the provided 'SQLite3_result*'. * @details The expected contents of the provided 'SQLite3_result*' are the ones generated by @@ -3505,13 +3497,67 @@ unique_ptr init_mmsd_with_conn( return mmsd; } +using gr_srv_addr_t = pair; + struct gr_srv_st_t { bool viable_candidate = false; bool read_only = true; int64_t transactions_behind = -1; bool inv_srv_state = false; + vector gr_members {}; }; +#define GR_MEMBER_ENTRY_ERR "%s '%s' in 'members' field from GR query to server '%s:%d'. Autodiscovery action aborted.\n" + +vector> parse_gr_members_addrs( + const MySQL_Monitor_State_Data* mmsd, const vector& gr_cluster_members +) { +#ifdef DEBUG + nlohmann::ordered_json members { gr_cluster_members }; + proxy_debug( + PROXY_DEBUG_MONITOR, 7, "Received 'members' field '%s' from GR query to server '%s:%d'\n", members.dump().c_str(), + mmsd->hostname, mmsd->port + ); +#endif + vector> result {}; + + for (const auto& cluster_member : gr_cluster_members) { + const vector gr_member_host_port { split_str(cluster_member, ':') }; + if (gr_member_host_port.size() != 2) { + proxy_error(GR_MEMBER_ENTRY_ERR, "Invalid server entry", cluster_member.c_str(), mmsd->hostname, mmsd->port); + break; + } + + const string srv_host { gr_member_host_port[0] }; + const char* c_str_port { gr_member_host_port[1].c_str() }; + + int32_t srv_port = -1; + + { + char* p_end = nullptr; + long port = std::strtol(c_str_port, &p_end, 10); + + if (c_str_port == p_end) { + proxy_error( + GR_MEMBER_ENTRY_ERR, "Failed to parse port for server entry", cluster_member.c_str(), mmsd->hostname, mmsd->port + ); + break; + } else { + srv_port = port; + } + } + + result.push_back({srv_host, srv_port}); + } + + // If any entry fails to parse, we invalidate the whole action + if (gr_cluster_members.size() != result.size()) { + return {}; + } else { + return result; + } +} + gr_srv_st_t extract_gr_srv_st(MySQL_Monitor_State_Data* mmsd) { gr_srv_st_t gr_srv_st {}; @@ -3522,7 +3568,7 @@ gr_srv_st_t extract_gr_srv_st(MySQL_Monitor_State_Data* mmsd) { num_fields = mysql_num_fields(mmsd->result); num_rows = mysql_num_rows(mmsd->result); - if (fields == NULL || num_fields!=3 || num_rows!=1) { + if (fields == NULL || num_fields!=4 || num_rows!=1) { proxy_error( "'mysql_fetch_fields' returns 'NULL', or 'mysql_num_fields(%d)', or 'mysql_num_rows(%d)' are incorrect." " Server %s:%d. See bug #1994\n", @@ -3543,11 +3589,17 @@ gr_srv_st_t extract_gr_srv_st(MySQL_Monitor_State_Data* mmsd) { if (row[2]) { gr_srv_st.transactions_behind=atol(row[2]); } + if (mmsd->cur_monitored_gr_srvs && row[3]) { + const string str_members_addrs { row[3] }; + const vector members_addrs { split_str(str_members_addrs, ',') }; + + gr_srv_st.gr_members = parse_gr_members_addrs(mmsd, members_addrs); + } } } proxy_debug( - PROXY_DEBUG_MONITOR, 4, + PROXY_DEBUG_MONITOR, 7, "Fetched %u:%s:%d info - interr: %d, error: %s, viable_candidate:'%d', read_only:'%d'," " transactions_behind:'%ld'\n", mmsd->hostgroup_id, mmsd->hostname, mmsd->port, mmsd->interr, mmsd->mysql_error_msg, @@ -3578,7 +3630,6 @@ gr_node_info_t gr_update_hosts_map( // NOTE: This isn't specified in the initializer list due to current standard limitations gr_node_info_t node_info {}; node_info.srv_st = gr_srv_st; - MySQL_Monitor_State_Data_Task_Result task_result = mmsd->get_task_result(); // Consider 'time_now' to be 'now - fetch_duration' unsigned long long time_now=realtime_time(); @@ -3671,6 +3722,24 @@ void gr_mon_action_over_resp_srv(MySQL_Monitor_State_Data* mmsd, const gr_node_i MyHGM->group_replication_lag_action( mmsd->writer_hostgroup, mmsd->hostname, mmsd->port, node_info.lag_counts, node_info.srv_st.read_only, enable ); + + if (mmsd->cur_monitored_gr_srvs && node_info.srv_st.gr_members.empty() == false) { + for (const gr_srv_addr_t& gr_member : node_info.srv_st.gr_members) { + const string& srv_host { gr_member.first }; + const int32_t srv_port { gr_member.second }; + bool found = false; + + for (const gr_host_def_t& host_def : *mmsd->cur_monitored_gr_srvs) { + if (srv_host == host_def.host && srv_port == host_def.port) { + found = true; + } + } + + if (found == false) { + MyHGM->update_group_replication_add_autodiscovered(srv_host, srv_port, mmsd->writer_hostgroup); + } + } + } } } } @@ -3905,6 +3974,11 @@ void* monitor_GR_thread_HG(void *arg) { } } + int rnd_discoverer = conn_mmsds.size() == 0 ? -1 : rand() % conn_mmsds.size(); + if (rnd_discoverer != -1) { + conn_mmsds[rnd_discoverer]->cur_monitored_gr_srvs = &hosts_defs; + } + // TODO: This needs to be reworked once we change the way monitoring actions work on clusters, taking // the full cluster fetch data to avoid transient states. For now, since we perform the monitoring // actions independently, we workaround the limitation of 'Monitor_Poll' of only handling @@ -3934,6 +4008,10 @@ void* monitor_GR_thread_HG(void *arg) { /////////////////////////////////////////////////////////////////////////////////////// + if (rnd_discoverer != -1) { + conn_mmsds[rnd_discoverer]->cur_monitored_gr_srvs = nullptr; + } + // Set the time for the next iteration next_check_time = curtime + mysql_thread___monitor_groupreplication_healthcheck_interval * 1000; } diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index 1207a04ef7..2c20fcc7d4 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -362,6 +362,18 @@ int wexecvp( return child_err; } +std::vector split_str(const std::string& s, char delimiter) { + std::vector tokens {}; + std::string token {}; + std::istringstream tokenStream(s); + + while (std::getline(tokenStream, token, delimiter)) { + tokens.push_back(token); + } + + return tokens; +} + std::string replace_str(const std::string& str, const std::string& match, const std::string& repl) { if(match.empty()) { return str; diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 6d7e606e7a..c22cca2957 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -751,7 +751,7 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p // NOTE: This query should be in one place that can be reused by // 'ProxySQL_Monitor' module. const std::string grouprep_monitor_test_query_start { - "SELECT viable_candidate,read_only,transactions_behind " + "SELECT viable_candidate,read_only,transactions_behind,members " "FROM GR_MEMBER_ROUTING_CANDIDATE_STATUS " }; @@ -769,14 +769,15 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p free(query); std::string t_select_as_query { - "SELECT '%s' AS viable_candidate, '%s' AS read_only, %d AS transactions_behind" + "SELECT '%s' AS viable_candidate, '%s' AS read_only, %d AS transactions_behind, '%s' AS members" }; std::string select_as_query {}; string_format( t_select_as_query, select_as_query, std::get<0>(gr_srv_status) ? "YES" : "NO", std::get<1>(gr_srv_status) ? "YES" : "NO", - std::get<2>(gr_srv_status) + std::get<2>(gr_srv_status), + std::get<3>(gr_srv_status).c_str() ); query = static_cast(malloc(select_as_query.length() + 1)); @@ -913,7 +914,7 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p #ifdef TEST_GROUPREP group_rep_status SQLite3_Server::grouprep_test_value(const std::string& srv_addr) { - group_rep_status cur_srv_st { "YES", "YES", 0 }; + group_rep_status cur_srv_st { "YES", "YES", 0, "" }; auto it = grouprep_map.find(srv_addr); if (it != grouprep_map.end()) { @@ -1471,7 +1472,8 @@ void SQLite3_Server::populate_grouprep_table(MySQL_Session *sess, int txs_behind const group_rep_status srv_status { std::string { r->fields[2] } == "YES" ? true : false, std::string { r->fields[3] } == "YES" ? true : false, - atoi(r->fields[4]) + atoi(r->fields[4]), + std::string { r->fields[5] } }; this->grouprep_map[srv_addr] = srv_status; @@ -1498,16 +1500,16 @@ void SQLite3_Server::populate_grouprep_table(MySQL_Session *sess, int txs_behind int hostgroup_id = atoi(r->fields[2]); const std::string t_insert_query { "INSERT INTO GR_MEMBER_ROUTING_CANDIDATE_STATUS" - " (hostname, port, viable_candidate, read_only, transactions_behind) VALUES" - " ('%s', %d, '%s', '%s', 0)" + " (hostname, port, viable_candidate, read_only, transactions_behind, members) VALUES" + " ('%s', %d, '%s', '%s', 0, '%s')" }; std::string insert_query {}; if (hostgroup_id % 4 == 0) { - string_format(t_insert_query, insert_query, hostname.c_str(), port, "YES", "NO"); + string_format(t_insert_query, insert_query, hostname.c_str(), port, "YES", "NO", ""); sessdb->execute(insert_query.c_str()); } else { - string_format(t_insert_query, insert_query, hostname.c_str(), port, "YES", "YES"); + string_format(t_insert_query, insert_query, hostname.c_str(), port, "YES", "YES", ""); sessdb->execute(insert_query.c_str()); } } @@ -1588,7 +1590,7 @@ bool SQLite3_Server::init() { insert_into_tables_defs(tables_defs_grouprep, (const char *)"GR_MEMBER_ROUTING_CANDIDATE_STATUS", (const char*)"CREATE TABLE GR_MEMBER_ROUTING_CANDIDATE_STATUS (" - "hostname VARCHAR NOT NULL, port INT NOT NULL, viable_candidate varchar not null, read_only varchar not null, transactions_behind int not null, PRIMARY KEY (hostname, port)" + "hostname VARCHAR NOT NULL, port INT NOT NULL, viable_candidate varchar not null, read_only varchar not null, transactions_behind int not null, members VARCHAR NOT NULL, PRIMARY KEY (hostname, port)" ")" ); From 4ee5c8bdde04daa3263ed9e6773b6d3293da947b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 8 May 2023 17:05:53 +0200 Subject: [PATCH 21/56] Improve recovery (OFFLINE_HARD) optimization for GR autodiscovered servers In case an autodiscovered server is found 'OFFLINE_HARD', the recovery logic now properly updates the 'servers_defaults' values for the server, in the same way as for a new discovered server. --- include/MySQL_HostGroups_Manager.h | 6 ++-- lib/MySQL_HostGroups_Manager.cpp | 48 +++++++++++++++++++----------- lib/MySQL_Monitor.cpp | 15 +++++++++- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 92a3add129..e370e926d8 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -828,9 +828,11 @@ class MySQL_HostGroups_Manager { * @details For adding the new server, several actions are performed: * 1. Lookup the target server in the corresponding MyHGC for the supplied hostgroup. * 2. If server is found, and it's status isn't 'OFFLINE_HARD' do nothing. Otherwise: - * - If server is found as 'OFFLINE_HARD', re-enable the server, log the action. + * - If server is found as 'OFFLINE_HARD', reset the internal values corresponding to + * 'servers_defaults' values to '-1', update the defaulted values to the ones in its 'MyHGC', lastly + * re-enable the server and log the action. * - If server isn't found, create it in the corresponding reader hostgroup of the supplied writer - * hostgroup, setting all 'servers_defaults' params as '-1', log the action.pasalo + * hostgroup, setting all 'servers_defaults' params as '-1', log the action. * - After any of the two previous actions, always regenerate servers data structures. * * NOTE: Server data structures regeneration requires: diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 929203536f..352da1ccac 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3427,23 +3427,13 @@ inline double get_prometheus_counter_val( return current_val; } -void MySQL_HostGroups_Manager::add(MySrvC *mysrvc, unsigned int _hid) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Adding MySrvC %p (%s:%d) for hostgroup %d\n", mysrvc, mysrvc->address, mysrvc->port, _hid); - - // Since metrics for servers are stored per-endpoint; the metrics for a particular endpoint can live longer than the - // 'MySrvC' itself. For example, a failover or a server config change could remove the server from a particular - // hostgroup, and a subsequent one bring it back to the original hostgroup. For this reason, everytime a 'mysrvc' is - // created and added to a particular hostgroup, we update the endpoint metrics for it. - std::string endpoint_id { std::to_string(_hid) + ":" + string { mysrvc->address } + ":" + std::to_string(mysrvc->port) }; - - mysrvc->bytes_recv = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_recv_map, endpoint_id); - mysrvc->bytes_sent = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_sent_map, endpoint_id); - mysrvc->connect_ERR = get_prometheus_counter_val(this->status.p_connection_pool_conn_err_map, endpoint_id); - mysrvc->connect_OK = get_prometheus_counter_val(this->status.p_connection_pool_conn_ok_map, endpoint_id); - mysrvc->queries_sent = get_prometheus_counter_val(this->status.p_connection_pool_queries_map, endpoint_id); - - MyHGC *myhgc=MyHGC_lookup(_hid); +void reset_hg_attrs_server_defaults(MySrvC* mysrvc) { + mysrvc->weight = -1; + mysrvc->max_connections = -1; + mysrvc->use_ssl = -1; +} +void update_hg_attrs_server_defaults(MySrvC* mysrvc, MyHGC* myhgc) { if (mysrvc->weight == -1) { if (myhgc->servers_defaults.weight != -1) { mysrvc->weight = myhgc->servers_defaults.weight; @@ -3468,7 +3458,25 @@ void MySQL_HostGroups_Manager::add(MySrvC *mysrvc, unsigned int _hid) { mysrvc->use_ssl = 0; } } +} +void MySQL_HostGroups_Manager::add(MySrvC *mysrvc, unsigned int _hid) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Adding MySrvC %p (%s:%d) for hostgroup %d\n", mysrvc, mysrvc->address, mysrvc->port, _hid); + + // Since metrics for servers are stored per-endpoint; the metrics for a particular endpoint can live longer than the + // 'MySrvC' itself. For example, a failover or a server config change could remove the server from a particular + // hostgroup, and a subsequent one bring it back to the original hostgroup. For this reason, everytime a 'mysrvc' is + // created and added to a particular hostgroup, we update the endpoint metrics for it. + std::string endpoint_id { std::to_string(_hid) + ":" + string { mysrvc->address } + ":" + std::to_string(mysrvc->port) }; + + mysrvc->bytes_recv = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_recv_map, endpoint_id); + mysrvc->bytes_sent = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_sent_map, endpoint_id); + mysrvc->connect_ERR = get_prometheus_counter_val(this->status.p_connection_pool_conn_err_map, endpoint_id); + mysrvc->connect_OK = get_prometheus_counter_val(this->status.p_connection_pool_conn_ok_map, endpoint_id); + mysrvc->queries_sent = get_prometheus_counter_val(this->status.p_connection_pool_queries_map, endpoint_id); + + MyHGC *myhgc=MyHGC_lookup(_hid); + update_hg_attrs_server_defaults(mysrvc, myhgc); myhgc->mysrvs->add(mysrvc); } @@ -5671,10 +5679,16 @@ void MySQL_HostGroups_Manager::update_group_replication_add_autodiscovered( for (uint32_t j = 0; j < myhgc->mysrvs->cnt(); j++) { MySrvC* mysrvc = static_cast(myhgc->mysrvs->servers->index(j)); - // TODO: Motivation for this logic needs to be better described + // If the server is found as 'OFFLINE_HARD' we reset the 'MySrvC' values corresponding with the + // 'servers_defaults' (as in a new 'MySrvC' creation). We then later update these values with the + // 'servers_defaults' attributes from its corresponding 'MyHGC'. This way we ensure uniform behavior + // of new servers, and 'OFFLINE_HARD' ones when a user update 'servers_defaults' values, and reloads + // the servers to runtime. if (strcmp(mysrvc->address,_host.c_str())==0 && mysrvc->port==_port) { srv_found = true; if (mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + reset_hg_attrs_server_defaults(mysrvc); + update_hg_attrs_server_defaults(mysrvc, mysrvc->myhgc); proxy_info( "Found healthy previously discovered GR node %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:" " hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index efaa9a1dab..9262c4ff2e 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -235,7 +235,20 @@ class MySQL_Monitor_Connection_Pool { #endif // DEBUG return; }; - // TODO: Document 'unregister' flow related async fetching and 'task_handlers'. + /** + * @brief Unregister the conn from the supplied 'mmsd'. + * @details DEBUG only helper function useful for checking the get/put connection flow + * for 'MySQL_Monitor_Connection_Pool'. This function should be called whenever a monitoring action does + * no longer require the conn of it's 'MMSD' and the conn has been considered 'non-suited' for being + * returned to the conn pool. This can be due to a failure in the data querying from the server itself, + * or due to unexpected data retrieved from the server. Due to this, the flow for calling this function + * during 'async' monitoring actions is: + * - If an error has taken place during the fetching itself, this function shall be called as soon as + * the failure is detected by the async state machine. + * - In case no error has taken place (TASK_RESULT_SUCCESS), this function should be called by the + * task-handler if it determines that the retrieved data is malformed. See handle_mmsd_mysql_conn. + * @param mmsd The 'mmsd' which conn should be unregistered. + */ void conn_unregister(MySQL_Monitor_State_Data *mmsd) { #ifdef DEBUG std::lock_guard lock(mutex); From 549a8280cf16d49fe469947ba6db93af65ad8d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 9 May 2023 12:38:48 +0200 Subject: [PATCH 22/56] Improve error reporting for invalid monitoring GR resultsets --- lib/MySQL_Monitor.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 9262c4ff2e..ca85d2c78c 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -3582,13 +3582,20 @@ gr_srv_st_t extract_gr_srv_st(MySQL_Monitor_State_Data* mmsd) { num_rows = mysql_num_rows(mmsd->result); if (fields == NULL || num_fields!=4 || num_rows!=1) { - proxy_error( - "'mysql_fetch_fields' returns 'NULL', or 'mysql_num_fields(%d)', or 'mysql_num_rows(%d)' are incorrect." - " Server %s:%d. See bug #1994\n", - num_fields, num_rows, mmsd->hostname, mmsd->port - ); + if (num_rows == 0) { + proxy_error( + "Empty resultset for GR monitoring query from server %s:%d. Server is likely misconfigured\n", + mmsd->hostname, mmsd->port + ); + } else { + proxy_error( + "Invalid resultset for GR monitoring query from server %s:%d. Either 'mysql_fetch_fields=NULL' or unexpected 'mysql_num_fields=%d'." + " Please report this incident\n", + mmsd->hostname, mmsd->port, num_fields + ); + } if (mmsd->mysql_error_msg == NULL) { - mmsd->mysql_error_msg = strdup("Unknown error"); + mmsd->mysql_error_msg = strdup("Invalid or malformed resultset"); } gr_srv_st.inv_srv_state = true; } else { @@ -3861,7 +3868,7 @@ void gr_report_fetching_errs(MySQL_Monitor_State_Data* mmsd) { */ void async_gr_mon_actions_handler(MySQL_Monitor_State_Data* mmsd) { // We base 'start_time' on the conn init for 'MySQL_Monitor_State_Data'. If a conn creation was - // required, we take into account this time into account, otherwise we asume that 'start_time=t1'. + // required, we take into account this time into account, otherwise we assume that 'start_time=t1'. uint64_t start_time = 0; if (mmsd->created_conn) { start_time = mmsd->init_time; From 4d9e210eb28611c7449371d63b9168396eef126e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 25 May 2023 09:59:56 +0200 Subject: [PATCH 23/56] WIP: Initial commit adding support for bootstrap mode - Introduced multiple new command line options - Gave additional meaning to 'mysql-default_collation_connection' --- include/MySQL_HostGroups_Manager.h | 43 ++ include/proxysql_admin.h | 25 +- include/proxysql_glovars.hpp | 18 + include/proxysql_utils.h | 8 + lib/MySQL_HostGroups_Manager.cpp | 43 -- lib/MySQL_Protocol.cpp | 2 +- lib/MySQL_Thread.cpp | 33 +- lib/ProxySQL_Admin.cpp | 467 +++++++++++++++++++- lib/ProxySQL_GloVars.cpp | 141 +++++- lib/proxysql_utils.cpp | 27 ++ src/main.cpp | 661 ++++++++++++++++++++++++++++- 11 files changed, 1409 insertions(+), 59 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index e370e926d8..1d32b6a3e5 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -904,4 +904,47 @@ class MySQL_HostGroups_Manager { MySrvC* find_server_in_hg(unsigned int _hid, const std::string& addr, int port); }; +/** + * @brief Helper function used to try to extract a value from the JSON field 'servers_defaults'. + * + * @param j JSON object constructed from 'servers_defaults' field. + * @param hid Hostgroup for which the 'servers_defaults' is defined in 'mysql_hostgroup_attributes'. Used for + * error logging. + * @param key The key for the value to be extracted. + * @param val_check A validation function, checks if the value is within a expected range. + * + * @return The value extracted from the supplied JSON. In case of error '-1', and error cause is logged. + */ +template ::value, bool>::type = true> +T j_get_srv_default_int_val( + const json& j, uint32_t hid, const string& key, const function& val_check +) { + if (j.find(key) != j.end()) { + const json::value_t val_type = j[key].type(); + const char* type_name = j[key].type_name(); + + if (val_type == json::value_t::number_integer || val_type == json::value_t::number_unsigned) { + T val = j[key].get(); + + if (val_check(val)) { + return val; + } else { + proxy_error( + "Invalid value %ld supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + static_cast(val), key.c_str(), hid + ); + } + } else { + proxy_error( + "Invalid type '%s'(%hhu) supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + type_name, static_cast(val_type), key.c_str(), hid + ); + } + } + + return static_cast(-1); +} + #endif /* __CLASS_MYSQL_HOSTGROUPS_MANAGER_H */ diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index d61d752893..e3610c81cd 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -114,6 +114,22 @@ struct admin_metrics_map_idx { }; }; +/** + * @brief Holds the retrieved info from the bootstrapping server. + * @details Used during ProxySQL_Admin initialization. + */ +struct bootstrap_info_t { + uint32_t server_language; + std::string server_version; + MYSQL_RES* servers; + MYSQL_RES* users; + std::string mon_user; + std::string mon_pass; + bool rand_gen_user; + + ~bootstrap_info_t(); +}; + // ProxySQL_Admin shared variables extern int admin__web_verbosity; @@ -344,7 +360,14 @@ class ProxySQL_Admin { #endif int pipefd[2]; void print_version(); - bool init(); + /** + * @brief Initializes the module. + * @details Bootstrap info is only used for 'bootstrap mode', i.e. if 'GloVars.global.gr_bootstrap_mode' + * is detected to be 'true'. + * @param bootstrap_info Info used to create default config during initialization in bootstrap mode. + * @return Always true. + */ + bool init(const bootstrap_info_t& bootstrap_info); void init_ldap(); bool get_read_only() { return variables.admin_read_only; } bool set_read_only(bool ro) { variables.admin_read_only=ro; return variables.admin_read_only; } diff --git a/include/proxysql_glovars.hpp b/include/proxysql_glovars.hpp index 70d02cafd9..93aefbb46e 100644 --- a/include/proxysql_glovars.hpp +++ b/include/proxysql_glovars.hpp @@ -121,6 +121,24 @@ class ProxySQL_GlobalVariables { #ifdef PROXYSQLCLICKHOUSE bool clickhouse_server; #endif /* PROXYSQLCLICKHOUSE */ + int gr_bootstrap_mode; + char* gr_bootstrap_uri; + char* gr_bootstrap_account; + char* gr_bootstrap_account_create; + char* gr_bootstrap_account_host; + uint64_t gr_bootstrap_password_retries; + char* gr_bootstrap_conf_bind_address; + uint64_t gr_bootstrap_conf_base_port; + bool gr_bootstrap_conf_use_sockets; + bool gr_bootstrap_conf_skip_tcp; + char* gr_bootstrap_ssl_ca; + char* gr_bootstrap_ssl_capath; + char* gr_bootstrap_ssl_cert; + char* gr_bootstrap_ssl_cipher; + char* gr_bootstrap_ssl_crl; + char* gr_bootstrap_ssl_crlpath; + char* gr_bootstrap_ssl_key; + char* gr_bootstrap_ssl_mode; pthread_mutex_t ext_glomth_mutex; } global; struct mysql { diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 76a1c2e302..5ccbb4eb7c 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -212,4 +212,12 @@ std::string replace_str(const std::string& str, const std::string& match, const std::vector split_str(const std::string& s, char delimiter); std::string generate_multi_rows_query(int rows, int params); + +/** + * @brief Generates a random string of the length of the provider 'strSize' parameter. + * @param strSize The size of the string to be generated. + * @return A random string. + */ +std::string rand_str(std::size_t strSize); + #endif diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 352da1ccac..7f69a0a12c 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -6965,49 +6965,6 @@ bool AWS_Aurora_Info::update(int r, int _port, char *_end_addr, int maxl, int al return ret; } -/** - * @brief Helper function used to try to extract a value from the JSON field 'servers_defaults'. - * - * @param j JSON object constructed from 'servers_defaults' field. - * @param hid Hostgroup for which the 'servers_defaults' is defined in 'mysql_hostgroup_attributes'. Used for - * error logging. - * @param key The key for the value to be extracted. - * @param val_check A validation function, checks if the value is within a expected range. - * - * @return The value extracted from the supplied JSON. In case of error '-1', and error cause is logged. - */ -template ::value, bool>::type = true> -T j_get_srv_default_int_val( - const json& j, uint32_t hid, const string& key, const function& val_check -) { - if (j.find(key) != j.end()) { - const json::value_t val_type = j[key].type(); - const char* type_name = j[key].type_name(); - - if (val_type == json::value_t::number_integer || val_type == json::value_t::number_unsigned) { - T val = j[key].get(); - - if (val_check(val)) { - return val; - } else { - proxy_error( - "Invalid value %ld supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." - " Value NOT UPDATED.\n", - static_cast(val), key.c_str(), hid - ); - } - } else { - proxy_error( - "Invalid type '%s'(%hhu) supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." - " Value NOT UPDATED.\n", - type_name, val_type, key.c_str(), hid - ); - } - } - - return static_cast(-1); -} - /** * @brief Initializes the supplied 'MyHGC' with the specified 'servers_defaults'. * @details Input verification is performed in the supplied 'server_defaults'. It's expected to be a valid diff --git a/lib/MySQL_Protocol.cpp b/lib/MySQL_Protocol.cpp index 589f4e8954..6ee1244207 100644 --- a/lib/MySQL_Protocol.cpp +++ b/lib/MySQL_Protocol.cpp @@ -1243,7 +1243,7 @@ bool MySQL_Protocol::generate_pkt_initial_handshake(bool send, void **ptr, unsig (*myds)->myconn->options.server_capabilities=mysql_thread___server_capabilities; memcpy(_ptr+l,&mysql_thread___server_capabilities, sizeof(mysql_thread___server_capabilities)/2); l+=sizeof(mysql_thread___server_capabilities)/2; const MARIADB_CHARSET_INFO *ci = NULL; - ci = proxysql_find_charset_name(mysql_thread___default_variables[SQL_CHARACTER_SET]); + ci = proxysql_find_charset_collate(mysql_thread___default_variables[SQL_COLLATION_CONNECTION]); if (!ci) { // LCOV_EXCL_START proxy_error("Cannot find character set for name [%s]. Configuration error. Check [%s] global variable.\n", diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index c70024fb7e..51f1ad4fef 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -116,15 +116,44 @@ const MARIADB_CHARSET_INFO * proxysql_find_charset_nr(unsigned int nr) { return NULL; } +/** + * @brief Finds the default (first) collation for the supplied 'charset name'. + * @details Previously, this function just returned the first collation found (default). Since v2.5.3, this + * function takes into consideration the thread variable 'SQL_COLLATION_CONNECTION' + * ('mysql-default_collation_connection'). This was introduced for being able to serve the same default + * collation as the server (for bootstrap mode) in case it's detected to be a MySQL 8 + * ('utf8mb4_0900_ai_ci'), instead of the retrocompatible default collation ('utf8mb4_general_ci'). This + * change also allows users to select the default collation that they please for a particular charset, if + * the collection specified via 'mysql-default_collation_connection', isn't found, the first found collation + * (original default) will be retrieved. + * @param name The 'charset name' for which to find the default collation. + * @return The collation found, NULL if none is find. + */ MARIADB_CHARSET_INFO * proxysql_find_charset_name(const char *name) { + const char* default_collation = mysql_thread___default_variables[SQL_COLLATION_CONNECTION]; MARIADB_CHARSET_INFO *c = (MARIADB_CHARSET_INFO *)mariadb_compiled_charsets; + MARIADB_CHARSET_INFO* charset_collation = nullptr; + do { if (!strcasecmp(c->csname, name)) { - return c; + if (charset_collation == nullptr) { + charset_collation = c; + } + + if (default_collation == nullptr) { + charset_collation = c; + break; + } else { + if (!strcmp(default_collation, c->name)) { + charset_collation = c; + break; + } + } } ++c; } while (c[0].nr != 0); - return NULL; + + return charset_collation; } MARIADB_CHARSET_INFO * proxysql_find_charset_collate_names(const char *csname, const char *collatename) { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 2e35084f9f..5fc7bdab8b 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7,6 +7,7 @@ #include #include #include "MySQL_HostGroups_Manager.h" +#include "mysql.h" #include "proxysql_admin.h" #include "re2/re2.h" #include "re2/regexp.h" @@ -1077,6 +1078,15 @@ incoming_servers_t::incoming_servers_t( incoming_hostgroup_attributes(incoming_hostgroup_attributes) {} +bootstrap_info_t::~bootstrap_info_t() { + if (servers != nullptr) { + mysql_free_result(servers); + } + if (users != nullptr) { + mysql_free_result(users); + } +} + int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { int r = 0; if (!GloQPro) return 0; @@ -5972,7 +5982,269 @@ void ProxySQL_Admin::init_ldap() { } } -bool ProxySQL_Admin::init() { +struct boot_srv_info_t { + string member_id; + string member_host; + uint32_t member_port; + string member_state; + string member_role; + string member_version; +}; + +struct BOOT_SRV_INFO_T { + enum { + MEMBER_ID, + MEMBER_HOST, + MEMBER_PORT, + MEMBER_STATE, + MEMBER_ROLE, + MEMBER_VERSION + }; +}; + +struct boot_user_info_t { + string user; + string ssl_type; + string auth_string; + string auth_plugin; + bool password_expired; +}; + +struct BOOT_USER_INFO_T { + enum { + USER, + SSL_TYPE, + AUTH_STRING, + AUTH_PLUGIN, + PASSWORD_EXPIRED + }; +}; + +struct srv_defs_t { + int64_t weight; + int64_t max_conns; + int32_t use_ssl; +}; + +using boot_srv_cnf_t = pair; + +vector extract_boot_servers_info(MYSQL_RES* servers) { + vector servers_info {}; + + while (MYSQL_ROW row = mysql_fetch_row(servers)) { + servers_info.push_back({ + string { row[BOOT_SRV_INFO_T::MEMBER_ID] }, + string { row[BOOT_SRV_INFO_T::MEMBER_HOST] }, + static_cast(stoi(row[BOOT_SRV_INFO_T::MEMBER_PORT])), + string { row[BOOT_SRV_INFO_T::MEMBER_STATE] }, + string { row[BOOT_SRV_INFO_T::MEMBER_ROLE] }, + string { row[BOOT_SRV_INFO_T::MEMBER_VERSION] }, + }); + } + + return servers_info; +} + +string build_boot_servers_insert(const vector& srvs_info_defs) { + const string t_srvs_insert { + "INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,max_connections,use_ssl) VALUES " + }; + string t_srvs_values {}; + + for (const auto& info_defs : srvs_info_defs) { + const boot_srv_info_t& srv_info { info_defs.first }; + const srv_defs_t& srv_defs { info_defs.second }; + + const char t_values[] { "(%d, \"%s\", %d, \"%s\", %ld, %ld, %d)" }; + string srv_values = cstr_format( + t_values, + srv_info.member_role == "PRIMARY" ? 0 : 1, // HOSTGROUP_ID + srv_info.member_host.c_str(), // HOSTNAME + srv_info.member_port, // PORT + srv_info.member_state.c_str(), // STATUS + srv_defs.weight, // Weight + srv_defs.max_conns, // Max Connections + srv_defs.use_ssl // UseSSL + ).str; + + if (&info_defs != &srvs_info_defs.back()) { + srv_values += ","; + } + + t_srvs_values += srv_values; + } + + const string servers_insert { t_srvs_insert + t_srvs_values }; + + return servers_insert; +} + +string build_boot_users_insert(MYSQL_RES* users) { + vector users_info {}; + + while (MYSQL_ROW row = mysql_fetch_row(users)) { + users_info.push_back({ + string { row[BOOT_USER_INFO_T::USER] }, + string { row[BOOT_USER_INFO_T::SSL_TYPE] }, + string { row[BOOT_USER_INFO_T::AUTH_STRING] }, + string { row[BOOT_USER_INFO_T::AUTH_PLUGIN] }, + static_cast(atoi(row[BOOT_USER_INFO_T::PASSWORD_EXPIRED])) + }); + } + + // MySQL Users + const string t_users_insert { + "INSERT INTO mysql_users (username,password,active,use_ssl) VALUES " + }; + string t_users_values {}; + + for (const boot_user_info_t& user : users_info) { + uint32_t use_ssl = user.ssl_type.empty() ? 0 : 1; + const char t_values[] { "(\"%s\", \"%s\", %d, %d)" }; + + string srv_values = cstr_format( + t_values, + user.user.c_str(), // USERNAME + user.auth_string.c_str(), // HOSTNAME + 1, // ACTIVE: Always ON + use_ssl // USE_SSL: Dependent on backend user + ).str; + + if (&user != &users_info.back()) { + srv_values += ","; + } + + t_users_values += srv_values; + } + + const string users_insert { t_users_insert + t_users_values }; + + return users_insert; +} + +map get_cur_hg_attrs(SQLite3DB* admindb) { + map res {}; + + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + admindb->execute_statement( + "SELECT hostgroup_id,servers_defaults FROM mysql_hostgroup_attributes", + &error, &cols, &affected_rows, &resultset + ); + + for (SQLite3_row* row : resultset->rows) { + const int32_t hid = atoi(row->fields[0]); + srv_defs_t srv_defs {}; + srv_defs.weight = 1; + srv_defs.max_conns = 512; + srv_defs.use_ssl = 1; + + nlohmann::json j_srv_defs = nlohmann::json::parse(row->fields[1]); + + const auto weight_check = [] (int64_t weight) -> bool { return weight >= 0; }; + srv_defs.weight = j_get_srv_default_int_val(j_srv_defs, hid, "weight", weight_check); + + const auto max_conns_check = [] (int64_t max_conns) -> bool { return max_conns >= 0; }; + srv_defs.max_conns = j_get_srv_default_int_val(j_srv_defs, hid, "max_connections", max_conns_check); + + const auto use_ssl_check = [] (int32_t use_ssl) -> bool { return use_ssl == 0 || use_ssl == 1; }; + srv_defs.use_ssl = j_get_srv_default_int_val(j_srv_defs, hid, "use_ssl", use_ssl_check); + + res.insert({ hid , srv_defs }); + } + + delete resultset; + + return res; +} + +vector build_srvs_info_with_defs( + const vector& srvs_info, + const map& hgid_defs, + const srv_defs_t global_defs +) { + vector res {}; + + for (const boot_srv_info_t& srv_info : srvs_info) { + if (srv_info.member_role == "PRIMARY") { + const auto hg_it = hgid_defs.find(0); + + if (hg_it != hgid_defs.end()) { + res.push_back({ srv_info, hg_it->second }); + } else { + res.push_back({ srv_info, global_defs }); + } + } else { + const auto hg_it = hgid_defs.find(1); + + if (hg_it != hgid_defs.end()) { + res.push_back({ srv_info, hg_it->second }); + } else { + res.push_back({ srv_info, global_defs }); + } + } + } + + return res; +} + +/** + * @brief Helper function used to check if tables are already filled with data. + * @details Handles the boilerplate operations of executing 'SELECT COUNT(*)' alike queries. + * @param admindb An already initialized instance of a SQLite3DB object to 'mem_admindb'. + * @param query The query to be executed, it's required to be 'SELECT COUNT(*)' alike. + * @return The resulting int of the 'COUNT(*)' in case of success, '-1' otherwise. In case of error, error + * cause are logged, and `assert` is called. + */ +int check_if_user_config(SQLite3DB* admindb, const char* query) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error( + "Aborting due to failed query over SQLite3 - db: '%s', query: '%s', err: %s", admindb->get_url(), query, error + ); + assert(0); + } + + int count = -1; + + if (resultset != nullptr && !resultset->rows.empty() && resultset->rows[0]->cnt >= 0) { + const char* s_count = resultset->rows[0]->fields[0]; + char* p_end = nullptr; + + count = std::strtol(s_count, &p_end, 10); + + if (p_end == s_count || errno == ERANGE) { + proxy_error( + "Aborting due to invalid query output, expected single INT (E.g. 'COUNT(*)') - query: '%s'", query + ); + count = -1; + } + } + + if (count == -1) { + assert(0); + } + + delete resultset; + return count; +}; + +/** + * @brief Definition of an auxiliary table used to store bootstrap variables. + * @details Table is used only to store in configdb bootstrap variables that are required to persist between + * executions. + */ +#define ADMIN_SQLITE_TABLE_BOOTSTRAP_VARIABLES "CREATE TABLE IF NOT EXISTS bootstrap_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" + +bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { cpu_timer cpt; Admin_HTTP_Server = NULL; @@ -6255,8 +6527,201 @@ bool ProxySQL_Admin::init() { __insert_or_replace_disktable_select_maintable(); } } + + /** + * @brief Inserts a default 'mysql_group_replication_hostgroup'. + * @details Uses the following defaults: + * - writer_hostgroup: 0 + * - reader_hostgroup: 1 + * - backup_writer_hostgroup: 2 + * - offline_hostgroup: 3 + * - max_writers: 9 + * - writer_is_also_reader: 0 -> Keep hostgroups separated + * - max_transactions_behind: 0 + * + * The number of writers in 'multi_primary_mode' wont be restricted, user should tune this value to + * convenience. By default 'max_writers' is set to 9, as is the current member limitation for Group + * Replication. + */ + const char insert_def_gr_hgs[] { + "INSERT INTO mysql_group_replication_hostgroups (" + "writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers," + "writer_is_also_reader" + ") VALUES (0,2,1,3,1,9,0)" + }; + vector servers_info {}; + + if (GloVars.global.gr_bootstrap_mode) { + // Check if user config is present for 'mysql_group_replication_hostgroups' + bool user_gr_hg_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_group_replication_hostgroups"); + if (user_gr_hg_cnf == false) { + admindb->execute(insert_def_gr_hgs); + } else { + proxy_info("Bootstrap config, found previous user 'mysql_group_replication_hostgroups' config, reusing...\n"); + } + + // Stores current user config for 'mysql_hostgroup_attributes::servers_defaults' + map hgid_defs {}; + // Check if user config is present for 'mysql_hostgroup_attributes' + bool user_gr_hg_attrs_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_hostgroup_attributes"); + int32_t have_ssl = 1; + + // SSL explicitly disabled by user for backend connections + if (GloVars.global.gr_bootstrap_ssl_mode) { + if (strcasecmp(GloVars.global.gr_bootstrap_ssl_mode, "DISABLED") == 0) { + have_ssl = 0; + } + } + + const int64_t DEF_GR_SRV_WEIGHT = 1; + const int64_t DEF_GR_SRV_MAX_CONNS = 512; + const int32_t DEF_GR_SRV_USE_SSL = have_ssl; + + // Update 'mysql_hostgroup_attributes' with sensible defaults for the new discovered instances + if (user_gr_hg_attrs_cnf == false) { + const nlohmann::json j_def_attrs { + { "weight", DEF_GR_SRV_WEIGHT }, + { "max_connections", DEF_GR_SRV_MAX_CONNS }, + { "use_ssl", DEF_GR_SRV_USE_SSL } + }; + const string str_def_attrs { j_def_attrs.dump() }; + const string insert_def_hg_attrs { + "INSERT INTO mysql_hostgroup_attributes (hostgroup_id, servers_defaults) VALUES" + " (0,'"+ str_def_attrs + "'), (1,'" + str_def_attrs + "')" + }; + admindb->execute(insert_def_hg_attrs.c_str()); + } else { + proxy_info("Bootstrap config, found previous user 'mysql_hostgroup_attributes' config, reusing...\n"); + hgid_defs = get_cur_hg_attrs(admindb); + } + + // Define the 'global defaults'. Either pure defaults, or user specified (argument). These values are + // supersede if previous user config is found for 'mysql_hostgroup_attributes::servers_defaults'. + srv_defs_t global_srvs_defs {}; + global_srvs_defs.weight = DEF_GR_SRV_WEIGHT; + global_srvs_defs.max_conns = DEF_GR_SRV_MAX_CONNS; + global_srvs_defs.use_ssl = DEF_GR_SRV_USE_SSL; + + servers_info = extract_boot_servers_info(bootstrap_info.servers); + auto full_srvs_info = build_srvs_info_with_defs(servers_info, hgid_defs, global_srvs_defs); + const string servers_insert { build_boot_servers_insert(full_srvs_info) }; + + admindb->execute("DELETE FROM mysql_servers"); + admindb->execute(servers_insert.c_str()); + + const string users_insert { build_boot_users_insert(bootstrap_info.users) }; + admindb->execute("DELETE FROM mysql_users"); + admindb->execute(users_insert.c_str()); + + // Make the configuration persistent + flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); + flush_mysql_users__from_memory_to_disk(); + } + + // Admin variables 'bootstrap' modifications + if (GloVars.global.gr_bootstrap_mode) { + // TODO-NOTE: This MUST go away; 'admin-hash_passwords' will be deprecated + admindb->execute("UPDATE global_variables SET variable_value='false' WHERE variable_name='admin-hash_passwords'"); + } flush_admin_variables___database_to_runtime(admindb,true); + + if (GloVars.global.gr_bootstrap_mode) { + flush_admin_variables___runtime_to_database(configdb, false, true, false); + } + + // MySQL variables / MySQL Query Rules 'bootstrap' modifications + if (GloVars.global.gr_bootstrap_mode && !servers_info.empty()) { + const uint64_t base_port { + GloVars.global.gr_bootstrap_conf_base_port == 0 ? 6446 : + GloVars.global.gr_bootstrap_conf_base_port + }; + const string bind_addr { + GloVars.global.gr_bootstrap_conf_bind_address == nullptr ? "0.0.0.0" : + string { GloVars.global.gr_bootstrap_conf_bind_address } + }; + const string s_rw_port { std::to_string(base_port) }; + const string s_ro_port { std::to_string(base_port + 1) }; + const string rw_addr { bind_addr + ":" + s_rw_port }; + const string ro_addr { bind_addr + ":" + s_ro_port }; + const string mysql_interfaces { rw_addr + ";" + ro_addr }; + + // Look for the default collation + const MARIADB_CHARSET_INFO* charset_info = proxysql_find_charset_nr(bootstrap_info.server_language); + const char* server_charset = charset_info == nullptr ? "" : charset_info->csname; + const char* server_collation = charset_info == nullptr ? "" : charset_info->name; + + // Holds user specified values, defaults, and implications of variables over others + const map bootstrap_mysql_vars { + { "mysql-server_version", bootstrap_info.server_version.c_str() }, + { "mysql-default_charset", server_charset }, + { "mysql-default_collation_connection", server_collation }, + { "mysql-interfaces", mysql_interfaces.c_str() }, + { "mysql-monitor_username", bootstrap_info.mon_user.c_str() }, + { "mysql-monitor_password", bootstrap_info.mon_pass.c_str() }, + { "mysql-have_ssl", "true" }, + { "mysql-ssl_p2s_ca", GloVars.global.gr_bootstrap_ssl_ca }, + { "mysql-ssl_p2s_capath", GloVars.global.gr_bootstrap_ssl_capath }, + { "mysql-ssl_p2s_cert", GloVars.global.gr_bootstrap_ssl_cert }, + { "mysql-ssl_p2s_cipher", GloVars.global.gr_bootstrap_ssl_cipher }, + { "mysql-ssl_p2s_crl", GloVars.global.gr_bootstrap_ssl_crl }, + { "mysql-ssl_p2s_crlpath", GloVars.global.gr_bootstrap_ssl_crlpath }, + { "mysql-ssl_p2s_key", GloVars.global.gr_bootstrap_ssl_key } + }; + + for (const pair& p_var_val : bootstrap_mysql_vars) { + if (p_var_val.second != nullptr) { + const string& name { p_var_val.first }; + const string& value { p_var_val.second }; + const string update_mysql_var { + "UPDATE global_variables SET variable_value='" + value + "' WHERE variable_name='" + name + "'" + }; + + admindb->execute(update_mysql_var.c_str()); + } + } + + // MySQL Query Rules - Port based RW split + { + // TODO: This should be able to contain in the future Unix socket based rules + const string insert_rw_split_rules { + "INSERT INTO mysql_query_rules (rule_id,active,proxy_port,destination_hostgroup,apply) VALUES " + " (0,1," + s_rw_port + ",0,1), (1,1," + s_ro_port + ",1,1)" + }; + + // Preserve previous user config targeting hostgroups 0/1 + bool user_qr_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_query_rules"); + if (user_qr_cnf == false) { + admindb->execute(insert_rw_split_rules.c_str()); + } else { + proxy_info("Bootstrap config, found previous user 'mysql_query_rules' config, reusing...\n"); + } + + flush_GENERIC__from_to("mysql_query_rules", "memory_to_disk"); + } + + // Store the 'bootstrap_variables' + if (bootstrap_info.rand_gen_user) { + configdb->execute(ADMIN_SQLITE_TABLE_BOOTSTRAP_VARIABLES); + + const string insert_bootstrap_user { + "INSERT INTO bootstrap_variables (variable_name,variable_value) VALUES" + " ('bootstrap_username','" + string { bootstrap_info.mon_user } + "')" + }; + const string insert_bootstrap_pass { + "INSERT INTO bootstrap_variables (variable_name,variable_value) VALUES" + " ('bootstrap_password','" + string { bootstrap_info.mon_pass } + "')" + }; + + configdb->execute("DELETE FROM bootstrap_variables WHERE variable_name='bootstrap_username'"); + configdb->execute(insert_bootstrap_user.c_str()); + configdb->execute("DELETE FROM bootstrap_variables WHERE variable_name='bootstrap_password'"); + configdb->execute(insert_bootstrap_pass.c_str()); + } + } flush_mysql_variables___database_to_runtime(admindb,true); + if (GloVars.global.gr_bootstrap_mode) { + flush_mysql_variables___runtime_to_database(configdb, false, true, false); + } #ifdef PROXYSQLCLICKHOUSE flush_clickhouse_variables___database_to_runtime(admindb,true); #endif /* PROXYSQLCLICKHOUSE */ diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index 66cb8f8362..1f04043d72 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -93,6 +93,58 @@ ProxySQL_GlobalVariables::~ProxySQL_GlobalVariables() { checksums_values.mysql_variables.in_shutdown = true; checksums_values.ldap_variables.in_shutdown = true; checksums_values.proxysql_servers.in_shutdown = true; + if (global.gr_bootstrap_uri) { + free(global.gr_bootstrap_uri); + global.gr_bootstrap_uri = nullptr; + } + if (global.gr_bootstrap_account) { + free(global.gr_bootstrap_account); + global.gr_bootstrap_account = nullptr; + } + if (global.gr_bootstrap_account_create) { + free(global.gr_bootstrap_account_create); + global.gr_bootstrap_account_create = nullptr; + } + if (global.gr_bootstrap_account_host) { + free(global.gr_bootstrap_account_host); + global.gr_bootstrap_account_host = nullptr; + } + if (global.gr_bootstrap_conf_bind_address) { + free(global.gr_bootstrap_conf_bind_address); + global.gr_bootstrap_conf_bind_address = nullptr; + } + if (global.gr_bootstrap_ssl_ca) { + free(global.gr_bootstrap_ssl_ca); + global.gr_bootstrap_ssl_ca = nullptr; + } + if (global.gr_bootstrap_ssl_capath) { + free(global.gr_bootstrap_ssl_capath); + global.gr_bootstrap_ssl_capath = nullptr; + } + if (global.gr_bootstrap_ssl_cert) { + free(global.gr_bootstrap_ssl_cert); + global.gr_bootstrap_ssl_cert = nullptr; + } + if (global.gr_bootstrap_ssl_cipher) { + free(global.gr_bootstrap_ssl_cipher); + global.gr_bootstrap_ssl_cipher = nullptr; + } + if (global.gr_bootstrap_ssl_crl) { + free(global.gr_bootstrap_ssl_crl); + global.gr_bootstrap_ssl_crl = nullptr; + } + if (global.gr_bootstrap_ssl_crlpath) { + free(global.gr_bootstrap_ssl_crlpath); + global.gr_bootstrap_ssl_crlpath = nullptr; + } + if (global.gr_bootstrap_ssl_key) { + free(global.gr_bootstrap_ssl_key); + global.gr_bootstrap_ssl_key = nullptr; + } + if (global.gr_bootstrap_ssl_mode) { + free(global.gr_bootstrap_ssl_mode); + global.gr_bootstrap_ssl_mode = nullptr; + } }; ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : @@ -147,6 +199,24 @@ ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : #ifdef PROXYSQLCLICKHOUSE global.clickhouse_server=false; #endif /* PROXYSQLCLICKHOUSE */ + global.gr_bootstrap_mode = 0; + global.gr_bootstrap_uri = nullptr; + global.gr_bootstrap_account = nullptr; + global.gr_bootstrap_account_create = nullptr; + global.gr_bootstrap_account_host = nullptr; + global.gr_bootstrap_password_retries = 20; + global.gr_bootstrap_conf_bind_address = nullptr; + global.gr_bootstrap_conf_base_port = 0; + global.gr_bootstrap_conf_use_sockets = false; + global.gr_bootstrap_conf_skip_tcp = false; + global.gr_bootstrap_ssl_ca = nullptr; + global.gr_bootstrap_ssl_capath = nullptr; + global.gr_bootstrap_ssl_cert = nullptr; + global.gr_bootstrap_ssl_cipher = nullptr; + global.gr_bootstrap_ssl_crl = nullptr; + global.gr_bootstrap_ssl_crlpath = nullptr; + global.gr_bootstrap_ssl_key = nullptr; + global.gr_bootstrap_ssl_mode = nullptr; opt=new ez::ezOptionParser(); opt->overview="High Performance Advanced Proxy for MySQL"; opt->syntax="proxysql [OPTIONS]"; @@ -157,7 +227,8 @@ ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : opt->add((const char *)"",0,0,0,(const char *)"Display usage instructions.",(const char *)"-h",(const char *)"-help",(const char *)"--help",(const char *)"--usage"); opt->add((const char *)"",0,0,0,(const char *)"Print version",(const char *)"-V",(const char *)"--version"); #ifdef DEBUG - opt->add((const char *)"",0,1,0,(const char *)"Enable debugging messages with specific verbosity",(const char *)"-d",(const char *)"--debug"); + // NOTE: Temporal change for full 'bootstrap' compatibility, only '--debug' is allowed, '-d' is an alias for '-D' + opt->add((const char *)"",0,1,0,(const char *)"Enable debugging messages with specific verbosity",(const char *)"--debug"); #endif /* DEBUG */ opt->add((const char *)"",0,0,0,(const char *)"Starts only the admin service",(const char *)"-n",(const char *)"--no-start"); opt->add((const char *)"",0,0,0,(const char *)"Do not start Monitor Module",(const char *)"-M",(const char *)"--no-monitor"); @@ -168,6 +239,8 @@ ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : opt->add((const char *)"",0,0,0,(const char *)"Do not restart ProxySQL if crashes",(const char *)"-e",(const char *)"--exit-on-error"); opt->add((const char *)"~/proxysql.cnf",0,1,0,(const char *)"Configuration file",(const char *)"-c",(const char *)"--config"); opt->add((const char *)"",0,1,0,(const char *)"Datadir",(const char *)"-D",(const char *)"--datadir"); + // NOTE: Duplicated option for 'bootstrap' compatibility + opt->add((const char *)"",0,1,0,(const char *)"Datadir",(const char *)"-d",(const char *)"--directory"); opt->add((const char *)"",0,1,0,(const char *)"UUID",(const char *)"-U",(const char *)"--uuid"); opt->add((const char *)"",0,0,0,(const char *)"Rename/empty database file",(const char *)"--initial"); opt->add((const char *)"",0,0,0,(const char *)"Merge config file into database file",(const char *)"--reload"); @@ -178,6 +251,32 @@ ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : opt->add((const char *)"",0,1,0,(const char *)"Administration Unix Socket",(const char *)"-S",(const char *)"--admin-socket"); opt->add((const char *)"",0,0,0,(const char *)"Enable SQLite3 Server",(const char *)"--sqlite3-server"); + // Bootstrap General options + opt->add((const char *)"",0,1,0,(const char *)"Start ProxySQL in Group Replication bootstrap mode",(const char *)"-B", (const char *)"--bootstrap"); + opt->add((const char *)"",0,1,0, (const char *)"Account to use by monitoring after bootstrap, either reuses a specify account or creates a new one;" + " this behavior is controlled by related option '--acount-create'. When used, a password must be provided." ,(const char *)"--account"); + opt->add((const char *)"",0,1,0,(const char *)"Account creation policy for bootstrap. Possible values are:\n" + "- if-not-exists (default): If the account doesn't exist, create it, otherwise reuse it.\n" + "- always: Only bootstrap if the account isn't present and can be created.\n" + "- never: Only bootstrap if the account is already present.",(const char *)"--account-create"); + opt->add((const char *)"",0,1,0,(const char *)"Host pattern to be used for accounts created during bootstrap",(const char *)"--account-host"); + opt->add((const char *)"",0,1,0,(const char *)"Number of attempts for generating a password when creating an account during bootstrap",(const char *)"--password-retries"); + opt->add((const char *)"",0,1,0,(const char *)"Sets the default base port ('mysql-interfaces') for the default R/W split port based configuration",(const char *)"--conf-base-port"); + opt->add((const char *)"",0,1,0,(const char *)"Sets the default bind address ('mysql-interfaces'). Used in combination with '--conf-bind-port'",(const char *)"--conf-bind-address"); + // TODO: We should make query rules compatible with Unix socket domain addresses for routing + opt->add((const char *)"",0,1,0,(const char *)"bootstrap option, configures two Unix sockets with names 'mysql.sock' and 'mysqlro.sock'",(const char *)"--conf-use-sockets"); + opt->add((const char *)"",0,1,0,(const char *)"Sets the default base port for the default R/W split port based configuration",(const char *)"--conf-skip-tcp"); + // Bootstrap SSL options + opt->add((const char *)"",0,1,0,(const char *)"The path name of the Certificate Authority (CA) certificate file. Must specify the same certificate used by the server",(const char *)"--ssl-ca"); + opt->add((const char *)"",0,1,0,(const char *)"The path name of the directory that contains trusted SSL CA certificate files",(const char *)"--ssl-capath"); + opt->add((const char *)"",0,1,0,(const char *)"The path name of the client public key certificate file",(const char *)"--ssl-cert"); + opt->add((const char *)"",0,1,0,(const char *)"The list of permissible ciphers for SSL encryption",(const char *)"--ssl-cipher"); + opt->add((const char *)"",0,1,0,(const char *)"The path name of the file containing certificate revocation lists",(const char *)"--ssl-crl"); + opt->add((const char *)"",0,1,0,(const char *)"The path name of the directory that contains certificate revocation list files",(const char *)"--ssl-crlpath"); + opt->add((const char *)"",0,1,0,(const char *)"The path name of the client private key file",(const char *)"--ssl-key"); + // TODO: Complete information about this mode and it's relation with 'ssl-ca'. E.g: For 'VERIFY_CA' mode, + // MariaDB connector related options. Not direct option 'MYSQL_OPT_SSL_MODE'. + opt->add((const char *)"",0,1,0,(const char *)"SSL connection mode for using during bootstrap during normal operation with the backend servers. Only PREFERRED, and DISABLED are supported.",(const char *)"--ssl-mode"); #ifdef PROXYSQLCLICKHOUSE opt->add((const char *)"",0,0,0,(const char *)"Enable ClickHouse Server",(const char *)"--clickhouse-server"); #endif /* PROXYSQLCLICKHOUSE */ @@ -197,6 +296,21 @@ void ProxySQL_GlobalVariables::parse(int argc, const char * argv[]) { opt->parse(argc, argv); }; +void update_string_var_if_set(char** cur_val, ez::ezOptionParser* opt, const char* cmd_opt) { + if (opt->isSet(cmd_opt)) { + std::string val {}; + opt->get(cmd_opt)->getString(val); + if (*cur_val) { free(*cur_val); } + *cur_val = strdup(val.c_str()); + } +} + +void update_ulong_var_if_set(uint64_t& cur_val, ez::ezOptionParser* opt, const char* cmd_opt) { + if (opt->isSet(cmd_opt)) { + opt->get(cmd_opt)->getULong(cur_val); + } +} + void ProxySQL_GlobalVariables::process_opts_pre() { if (opt->isSet("-h")) { std::string usage; @@ -210,8 +324,8 @@ void ProxySQL_GlobalVariables::process_opts_pre() { exit(EXIT_SUCCESS); } - if (opt->isSet("-d")) { - opt->get("-d")->getInt(GloVars.__cmd_proxysql_gdbg); + if (opt->isSet("--debug")) { + opt->get("--debug")->getInt(GloVars.__cmd_proxysql_gdbg); global.gdbg=true; } @@ -235,6 +349,8 @@ void ProxySQL_GlobalVariables::process_opts_pre() { GloVars.__cmd_proxysql_datadir=strdup(datadir.c_str()); } + update_string_var_if_set(&GloVars.__cmd_proxysql_datadir, opt, "-d"); + if (opt->isSet("-U")) { std::string uuid; opt->get("-U")->getString(uuid); @@ -275,7 +391,24 @@ void ProxySQL_GlobalVariables::process_opts_pre() { global.clickhouse_server=true; } #endif /* PROXYSQLCLICKHOUSE */ - + update_string_var_if_set(&global.gr_bootstrap_uri, opt, "--bootstrap"); + global.gr_bootstrap_mode = opt->isSet("--bootstrap"); + update_ulong_var_if_set(global.gr_bootstrap_conf_base_port, opt, "--conf-base-port"); + update_string_var_if_set(&global.gr_bootstrap_conf_bind_address, opt, "--conf-bind-address"); + global.gr_bootstrap_conf_use_sockets = opt->isSet("--conf-use-sockets"); + global.gr_bootstrap_conf_skip_tcp = opt->isSet("--conf-skip-tcp"); + update_string_var_if_set(&global.gr_bootstrap_account, opt, "--account"); + update_string_var_if_set(&global.gr_bootstrap_account_create, opt, "--account-create"); + update_string_var_if_set(&global.gr_bootstrap_account_host, opt, "--account-host"); + update_ulong_var_if_set(global.gr_bootstrap_password_retries, opt, "--password-retries"); + update_string_var_if_set(&global.gr_bootstrap_ssl_ca, opt, "--ssl-ca"); + update_string_var_if_set(&global.gr_bootstrap_ssl_capath, opt, "--ssl-capath"); + update_string_var_if_set(&global.gr_bootstrap_ssl_cert, opt, "--ssl-cert"); + update_string_var_if_set(&global.gr_bootstrap_ssl_cipher, opt, "--ssl-cipher"); + update_string_var_if_set(&global.gr_bootstrap_ssl_crl, opt, "--ssl-crl"); + update_string_var_if_set(&global.gr_bootstrap_ssl_crlpath, opt, "--ssl-crlpath"); + update_string_var_if_set(&global.gr_bootstrap_ssl_key, opt, "--ssl-key"); + update_string_var_if_set(&global.gr_bootstrap_ssl_mode, opt, "--ssl-mode"); config_file=GloVars.__cmd_proxysql_config_file; diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index 2c20fcc7d4..f30f1e5b8d 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -1,10 +1,12 @@ #include "proxysql_utils.h" +#include #include #include #include #include +#include #include #include #include @@ -409,3 +411,28 @@ std::string generate_multi_rows_query(int rows, int params) { } return s; } + +string rand_str(std::size_t strSize) { + string dic { "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" }; + std::random_device rd {}; + std::mt19937 gen { rd() }; + + std::shuffle(dic.begin(), dic.end(), gen); + + if (strSize < dic.size()) { + return dic.substr(0, strSize); + } else { + std::size_t req_modulus = static_cast(strSize / dic.size()); + std::size_t req_reminder = strSize % dic.size(); + string random_str {}; + + for (std::size_t i = 0; i < req_modulus; i++) { + std::shuffle(dic.begin(), dic.end(), gen); + random_str.append(dic); + } + + random_str.append(dic.substr(0, req_reminder)); + + return random_str; + } +} diff --git a/src/main.cpp b/src/main.cpp index 24a53e2b7f..e1f9b558de 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4,6 +4,8 @@ #include "btree_map.h" #include "proxysql.h" +#include +#include #include #include #include @@ -11,6 +13,8 @@ //#define PROXYSQL_EXTERN #include "cpp.h" +#include "mysqld_error.h" + #include "ProxySQL_Statistics.hpp" #include "MySQL_PreparedStatement.h" #include "ProxySQL_Cluster.hpp" @@ -21,6 +25,7 @@ #include "MySQL_LDAP_Authentication.hpp" #include "proxysql_restapi.h" #include "Web_Interface.hpp" +#include "proxysql_utils.h" #include #include @@ -44,7 +49,9 @@ extern "C" MySQL_LDAP_Authentication * create_MySQL_LDAP_Authentication_func() { */ - +using std::map; +using std::string; +using std::vector; volatile create_MySQL_LDAP_Authentication_t * create_MySQL_LDAP_Authentication = NULL; @@ -561,6 +568,30 @@ void ProxySQL_Main_process_global_variables(int argc, const char **argv) { GloVars.ldap_auth_plugin=strdup(ldap_auth_plugin.c_str()); } } + const map varnames_globals_map { + { "mysql-ssl_p2s_ca", &GloVars.global.gr_bootstrap_ssl_ca }, + { "mysql-ssl_p2s_capath", &GloVars.global.gr_bootstrap_ssl_capath }, + { "mysql-ssl_p2s_cert", &GloVars.global.gr_bootstrap_ssl_cert }, + { "mysql-ssl_p2s_key", &GloVars.global.gr_bootstrap_ssl_key }, + { "mysql-ssl_p2s_cipher", &GloVars.global.gr_bootstrap_ssl_cipher }, + { "mysql-ssl_p2s_crl", &GloVars.global.gr_bootstrap_ssl_crl }, + { "mysql-ssl_p2s_crlpath", &GloVars.global.gr_bootstrap_ssl_crlpath } + }; + // Command line options always take precedence + if (GloVars.global.gr_bootstrap_mode && root.exists("mysql_variables")) { + const Setting& mysql_vars = root["mysql_variables"]; + + for (const pair& name_global : varnames_globals_map) { + for (const auto& setting_it : mysql_vars) { + if (*name_global.second == nullptr) { + if (setting_it.getName() == name_global.first && setting_it.isString()) { + const char* setting_val = setting_it.c_str(); + *name_global.second = strdup(setting_val); + } + } + } + } + } } else { proxy_warning("Unable to open config file %s\n", GloVars.config_file); // issue #705 if (GloVars.__cmd_proxysql_config_file) { @@ -670,7 +701,7 @@ void ProxySQL_Main_init_main_modules() { } -void ProxySQL_Main_init_Admin_module() { +void ProxySQL_Main_init_Admin_module(const bootstrap_info_t& bootstrap_info) { // cluster module needs to be initialized before GloProxyCluster = new ProxySQL_Cluster(); GloProxyCluster->init(); @@ -679,7 +710,7 @@ void ProxySQL_Main_init_Admin_module() { //GloProxyStats->init(); GloProxyStats->print_version(); GloAdmin = new ProxySQL_Admin(); - GloAdmin->init(); + GloAdmin->init(bootstrap_info); GloAdmin->print_version(); if (binary_sha1) { proxy_info("ProxySQL SHA1 checksum: %s\n", binary_sha1); @@ -932,7 +963,9 @@ void ProxySQL_Main_init() { static void LoadPlugins() { GloMyLdapAuth = NULL; - SQLite3DB::LoadPlugin(GloVars.sqlite3_plugin); + if (proxy_sqlite3_open_v2 == nullptr) { + SQLite3DB::LoadPlugin(GloVars.sqlite3_plugin); + } if (GloVars.web_interface_plugin) { dlerror(); char * dlsym_error = NULL; @@ -1008,11 +1041,11 @@ void UnloadPlugins() { } } -void ProxySQL_Main_init_phase2___not_started() { +void ProxySQL_Main_init_phase2___not_started(const bootstrap_info_t& boostrap_info) { LoadPlugins(); ProxySQL_Main_init_main_modules(); - ProxySQL_Main_init_Admin_module(); + ProxySQL_Main_init_Admin_module(boostrap_info); GloMTH->print_version(); { @@ -1315,6 +1348,342 @@ namespace { static const bool SET_TERMINATE = std::set_terminate(my_terminate); } +/** + * @brief Regex for parsing URI connections. + * @details Groups explanation: + * + HierPart doesn't hold '//' making previous non-capturing group optional. E.g: + * - '127.0.0.1:3306' + * - 'mysql-server-1:3306' + * + UserInfo is inside a non-capturing group to avoid matching '@'. + * + RegName matches any valid Ipv4 or domain name. + * + Ipv6 matches Ipv6, it's NOT spec conforming, we don't verify the supplied Ip in the regex. + * + Port matches the supplied port. + */ +const char CONN_URI_REGEX[] { + "^(:?(?P[a-z][a-z0-9\\+\\-\\.]*):\\/\\/)?" + "(?P" + "(?:(?P(?:\\%[0-9a-f][0-9a-f]|[a-z0-9\\-\\.\\_\\~]|[\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\=]|\\:)*)\\@)?" + "(?P" + "(?P(?:\\%[0-9a-f][0-9a-f]|[a-z0-9\\-\\.\\_\\~]|[\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\=]])*)|" + "(?P\\[(?:[0-9a-f]|[\\:])*\\])):" + "(?P[0-9]+))?" +}; + +/** + * @brief Holds each of the groups matched in a string by 'CONN_URI_REGEX'. + */ +struct conn_uri_t { + string scheme; + string hierpart; + string user; + string pass; + string host; + uint32_t port; +}; + +/** + * @brief Uses Regex 'CONN_URI_REGEX' to parse the supplied string. + * @details Tries to perform a 'PartialMatchN' over the 'CONN_URI_REGEX'. Right now doesn't perform a *full* + * check on the validity of the semantics of the URI itself. It does perform some checks. + * @param conn_uri A connection URI. + * @return On success `{EXIT_SUCCESS, conn_uri_t}`, otherwise `{EXIT_FAILURE, conn_uri_t{}}`. Error cause is + * logged. + */ +pair parse_conn_uri(const string& conn_uri) { + re2::RE2::Options opts(RE2::Quiet); + opts.set_case_sensitive(false); + + re2::RE2 re(CONN_URI_REGEX, opts);; + if (re.error_code()) { + proxy_error("Regex creation failed - %s\n", re.error().c_str()); + assert(0); + } + + const int num_groups = re.NumberOfCapturingGroups(); + std::vector str_args(num_groups, std::string {}); + std::vector re2_args {}; + + for (std::string& str_arg : str_args) { + re2_args.push_back(RE2::Arg(&str_arg)); + } + + std::vector matches {}; + for (RE2::Arg& re2_arg : re2_args) { + matches.push_back(&re2_arg); + } + + const re2::RE2::Arg* const* args = &matches[0]; + RE2::PartialMatchN(conn_uri, re, args, num_groups); + + const map& groups = re.NamedCapturingGroups(); + map::const_iterator group_it; + + string scheme {}; + string hierpart {}; + string userinfo {}; + string host {}; + uint32_t port = 0; + + if ((group_it = groups.find("Scheme")) != groups.end()) { scheme = str_args[group_it->second - 1]; } + if ((group_it = groups.find("HierPart")) != groups.end()) { hierpart = str_args[group_it->second - 1]; } + if ((group_it = groups.find("UserInfo")) != groups.end()) { userinfo = str_args[group_it->second - 1]; } + if ((group_it = groups.find("Host")) != groups.end()) { host = str_args[group_it->second - 1]; } + + // Remove the enclosing(`[]`) from IPv6 addresses + if (host.empty() == false && host.size() > 2) { + if (host[0] == '[') { + host = host.substr(1, host.size()-2); + } + } + + string user {}; + string pass {}; + + int32_t match_err = EXIT_SUCCESS; + + // Extract supplied info for user:pass + vector v_userinfo = split_str(userinfo, ':'); + if (v_userinfo.size() == 1) { + user = v_userinfo[0]; + } else if (v_userinfo.size() == 2) { + user = v_userinfo[0]; + pass = v_userinfo[1]; + } else if (v_userinfo.size() > 2) { + proxy_error( + "Invalid UserInfo '%s' supplied in connection URI. UserInfo should contain at max two fields 'user:pass'\n", + userinfo.c_str() + ); + match_err = EXIT_FAILURE; + } + + if ((group_it = groups.find("Port")) != groups.end()) { + const string s_port { str_args[group_it->second - 1] }; + + if (!s_port.empty()) { + char* p_end = nullptr; + port = std::strtol(s_port.c_str(), &p_end, 10); + + if (errno == ERANGE || p_end == s_port.c_str()) { + proxy_error("Invalid Port '%s' supplied in connection URI.\n", s_port.c_str()); + match_err = EXIT_FAILURE; + } + } + } + + struct conn_uri_t uri_data { scheme, hierpart, user, pass, host, port }; + + return { match_err, uri_data }; +} + +/** + * @brief Helper function to serialize 'conn_uri_t' for debugging purposes. + */ +string to_string(const conn_uri_t& conn_uri) { + nlohmann::ordered_json j; + + j["scheme"] = conn_uri.scheme; + j["user"] = conn_uri.user; + j["pass"] = conn_uri.pass; + j["host"] = conn_uri.host; + j["port"] = conn_uri.port; + + return j.dump(); +} + +/** + * @brief Query for fetching MySQL users during bootstrapping. + * @details For security reasons, users matching the following names are excluded: + * - `mysql.%` + * - `root` + * - `bt_proxysql_%` + * Users starting with `bt_proxysql_` are considered `ProxySQL` created used during `bootstrap` for + * monitoring purposes. A user, could create it's own monitoring accounts under this prefix, to avoid + * ProxySQL fetching them as regular users. + */ +const char BOOTSTRAP_SELECT_USERS[] { + "SELECT DISTINCT user,ssl_type,authentication_string,plugin,password_expired FROM mysql.user" + " WHERE user NOT LIKE 'mysql.%' AND user NOT LIKE 'bt_proxysql_%'" +#ifndef DEBUG + " AND user != 'root'"; +#endif +}; + +/** + * @brief Query for fetching MySQL servers during bootstrapping. + * @details As the regular GR monitoring queries, makes use of `replication_group_members` table. + */ +const char BOOTSTRAP_SELECT_SERVERS[] { + "SELECT MEMBER_ID,MEMBER_HOST,MEMBER_PORT,MEMBER_STATE,MEMBER_ROLE,MEMBER_VERSION FROM" + " performance_schema.replication_group_members" +}; + +/** + * @brief Stores credentials for monitoring created accounts during bootstrap. + */ +struct acct_creds_t { + string user; + string pass; +}; + +/** + * @brief Minimal set of permissions for a created GR monitoring account. + * @details Right now we **do not grant** permissions to `mysql_innodb_cluster_metadata` tables, since for now + * we don't make any use of them. + */ +const vector t_grant_perms_queries { + "GRANT USAGE ON *.* TO `%s`@`%%`", +// NOTE: For now, we don't make use of any `mysql_innodb_cluster_metadata` tables +// "GRANT SELECT, EXECUTE ON `mysql_innodb_cluster_metadata`.* TO `%s`@`%%`", +// NOTE: For now, we don't make use of 'routers' and 'v2_routers' table +// "GRANT INSERT, UPDATE, DELETE ON `mysql_innodb_cluster_metadata`.`routers` TO `%s`@`%%`", +// "GRANT INSERT, UPDATE, DELETE ON `mysql_innodb_cluster_metadata`.`v2_routers` TO `%s`@`%%`", + "GRANT SELECT ON `performance_schema`.`global_variables` TO `%s`@`%%`", + "GRANT SELECT ON `performance_schema`.`replication_group_member_stats` TO `%s`@`%%`", + "GRANT SELECT ON `performance_schema`.`replication_group_members` TO `%s`@`%%`" +}; + +/** + * @brief Grants the minimal set of permissions for GR monitoring to a supplies user. + * @details All permissions will be granted for host `%`. + * @param mysql An already opened MySQL connection. + * @param user The username to grant permissions to. + * @return Either `0` for success, or the corresponding `mysql_errno` for failure. + */ +int grant_user_perms(MYSQL* mysql, const string& user) { + for (const string& t_query : t_grant_perms_queries) { + const string query { cstr_format(t_query.c_str(), user.c_str()).str }; + + proxy_info("GRANT permissions '%s' to user\n", query.c_str()); + int myerr = mysql_query(mysql, query.c_str()); + if (myerr) { + return mysql_errno(mysql); + } + } + + return 0; +} + +/** + * @brief Generates a random password conforming with MySQL 'MEDIUM' policy. + * @param size The target password size. + * @return The random password generated. + */ +string gen_rand_password(std::size_t size) { + const string lowercase { "abcdefghijklmnopqrstuvwxyz" }; + const string uppercase { "ABCDEFGHIJKLMNOPQRSTUVWXYZ" }; + const string digits { "0123456789" }; + const string symbols { "~@#$^&*]}[{()|-=+;:.>,(remains / allphabet.size()); + std::size_t req_reminder = remains % allphabet.size(); + + for (std::size_t i = 0; i < req_modulus; i++) { + std::shuffle(allphabet.begin(), allphabet.end(), gen); + pass += allphabet; + } + + std::shuffle(allphabet.begin(), allphabet.end(), gen); + pass += allphabet.substr(0, req_reminder); + } + } + + return pass; +} + +/** + * @brief Creates a random monitoring account for bootstrap with a random generated password. + * @param mysql An already opened MySQL connection. + * @param max_retries Maximum number of attempts for creating the user. + * @return On success `{0, acct_creds_t}`, otherwise `{mysql_errno, acct_creds_t{}}`. Error cause is logged. + */ +pair create_random_bootstrap_account(MYSQL* mysql, uint32_t max_retries) { + // Random username + const string monitor_user { "bt_proxysql_" + rand_str(12) }; + string monitor_pass {}; + + int myerr = ER_NOT_VALID_PASSWORD; + uint32_t retries = 0; + + while (retries < max_retries && (myerr == ER_NOT_VALID_PASSWORD)) { + monitor_pass = gen_rand_password(16); + + const string user_create { + "CREATE USER IF NOT EXISTS '" + monitor_user + "'@'%' IDENTIFIED BY '" + monitor_pass + "'" + }; + + int myres = mysql_query(mysql, user_create.c_str()); + myerr = mysql_errno(mysql); + + if (myres || myerr) { + if (myerr != ER_NOT_VALID_PASSWORD) { + return { myerr, { "", "" } }; + } else { + proxy_info( + "Bootstrap config, failed to create password for user '%s'. Retrying...\n", monitor_user.c_str() + ); + retries += 1; + } + } else { + break; + } + } + + if (myerr == 0) { + myerr = grant_user_perms(mysql, monitor_user); + } + + return { myerr, { monitor_user, monitor_pass } }; +} + +/** + * @brief Creates a monitoring account for bootstrap with the supplied parameters. + * @param mysql An already opened MySQL connection. + * @param user The username for the new account. + * @param pass The password for the new account. + * @return On success `{0,acct_creds_t}`, otherwise `{mysql_errno,acct_creds_t}`. Doesn't log errors. + */ +pair create_bootstrap_account(MYSQL* mysql, const string& user, const string& pass) { + const string monitor_user { "'" + user + "'" }; + const string user_create { + "CREATE USER IF NOT EXISTS " + monitor_user + "@'%' IDENTIFIED BY '" + pass + "'" + }; + + int myerr = mysql_query(mysql, user_create.c_str()); + + if (myerr == 0) { + myerr = grant_user_perms(mysql, user); + } + + return { myerr, { user, pass } }; +} + int main(int argc, const char * argv[]) { { @@ -1364,6 +1733,284 @@ int main(int argc, const char * argv[]) { } } + bootstrap_info_t bootstrap_info {}; + // Try to connect to MySQL for performing the bootstrapping process: + // - If data isn't found we perform the bootstrap process. + // - If non-empty datadir is present, reconfiguration should be performed. + if (GloVars.global.gr_bootstrap_mode == 1) { + // Check the other required arguments for performing the bootstrapping process: + // - Username + // - Password - asked by prompt or supplied + // - Connection string parsing is required + const string conn_uri { GloVars.global.gr_bootstrap_uri }; + const pair parse_uri_res { parse_conn_uri(conn_uri) }; + const conn_uri_t uri_data { parse_uri_res.second }; + + if (parse_uri_res.first == EXIT_FAILURE) { + proxy_info("Aborting bootstrap due to failed to parse or match URI - `%s`\n", to_string(uri_data).c_str()); + exit(parse_uri_res.first); + } else { + proxy_info("Starting bootstrap connection with data extracted from URI - `%s`\n", to_string(uri_data).c_str()); + } + + const char* c_host = uri_data.host.c_str(); + const char* c_user = uri_data.user.empty() ? "root" : uri_data.user.c_str(); + const char* c_pass = nullptr; + uint32_t port = uri_data.port; + uint32_t flags = CLIENT_SSL; + + if (uri_data.pass.empty()) { + c_pass = getpass("Enter password: "); + } else { + c_pass = uri_data.pass.c_str(); + } + + MYSQL* mysql = mysql_init(NULL); + + // SSL explicitly disabled by user for backend connections + if (GloVars.global.gr_bootstrap_ssl_mode) { + if (!strcasecmp(GloVars.global.gr_bootstrap_ssl_mode, "DISABLED")) { + flags = 0; + } + } + + if (flags == CLIENT_SSL) { + mysql_ssl_set( + mysql, + GloVars.global.gr_bootstrap_ssl_key, + GloVars.global.gr_bootstrap_ssl_cert, + GloVars.global.gr_bootstrap_ssl_ca, + GloVars.global.gr_bootstrap_ssl_capath, + GloVars.global.gr_bootstrap_ssl_cipher + ); + } + + if (!mysql_real_connect(mysql, c_host, c_user, c_pass, nullptr, port, NULL, flags)) { + proxy_error("Bootstrap failed, connection error '%s'\n", mysql_error(mysql)); + exit(EXIT_FAILURE); + } + + if (uri_data.pass.empty()) { + uint32_t passlen = strlen(c_pass); + memset(static_cast(const_cast(c_pass)), 0, passlen); + } + + // Get server default collation and version directly from initial handshake + bootstrap_info.server_language = mysql->server_language; + bootstrap_info.server_version = mysql->server_version; + + // Fetch all required data for Bootstrap + int myrc = mysql_query(mysql, BOOTSTRAP_SELECT_SERVERS); + + if (myrc) { + proxy_error("Bootstrap failed, query failed with error - %s\n", mysql_error(mysql)); + exit(EXIT_FAILURE); + } + + MYSQL_RES* myres_members = mysql_store_result(mysql); + + if (myres_members == nullptr || mysql_num_rows(myres_members) == 0) { + proxy_error("Bootstrap failed, expected server %s:%d to have Group Replication configured\n", c_host, port); + exit(EXIT_FAILURE); + } + + myrc = mysql_query(mysql, BOOTSTRAP_SELECT_USERS); + + if (myrc) { + proxy_error("Bootstrap failed, query failed with error - %s\n", mysql_error(mysql)); + exit(EXIT_FAILURE); + } + + MYSQL_RES* myres_users = mysql_store_result(mysql); + + if (myres_users == nullptr) { + proxy_error("Bootstrap failed, storing resultset failed with error - %s\n", mysql_error(mysql)); + exit(EXIT_FAILURE); + } + + // TODO-NOTE: Maybe further data verification should be performed here; bootstrap-info holding final types + bootstrap_info.servers = myres_members; + bootstrap_info.users = myres_users; + + // Setup a bootstrap account - monitoring + const string account_create_policy { + GloVars.global.gr_bootstrap_account_create == nullptr ? "if-not-exists" : + GloVars.global.gr_bootstrap_account_create + }; + + if (GloVars.global.gr_bootstrap_account == nullptr && GloVars.global.gr_bootstrap_account_create != nullptr) { + proxy_error("Bootstrap failed, option '--account-create' can only be used in combination with '--account'\n"); + exit(EXIT_FAILURE); + } + + const uint32_t password_retries = GloVars.global.gr_bootstrap_password_retries; + string new_mon_user {}; + string new_mon_pass {}; + + if (GloVars.global.gr_bootstrap_account != nullptr) { + const vector valid_policies { "if-not-exists", "always", "never" }; + if (std::find(valid_policies.begin(), valid_policies.end(), account_create_policy) == valid_policies.end()) { + proxy_error("Bootstrap failed, unknown '--account-create' option '%s'\n", account_create_policy.c_str()); + exit(EXIT_FAILURE); + } + + // Since an account has been specified, we require the password for the account + const string mon_user { GloVars.global.gr_bootstrap_account }; + const string get_acc_pass_msg { "Please enter MySQL password for " + mon_user + ": " }; + + // Get the account pass directly from user input + const string mon_pass = getpass(get_acc_pass_msg.c_str()); + + // 1. Check if account exists + const string get_user_cnt { "SELECT COUNT(*) FROM mysql.user WHERE user='" + mon_user + "'" }; + int cnt_err = mysql_query(mysql, get_user_cnt.c_str()); + MYSQL_RES* myres = mysql_store_result(mysql); + + if (cnt_err || myres == nullptr) { + proxy_error("Bootstrap failed, detecting count existence failed with error - %s\n", mysql_error(mysql)); + exit(EXIT_FAILURE); + } + + MYSQL_ROW myrow = mysql_fetch_row(myres); + uint32_t acc_exists = atoi(myrow[0]); + mysql_free_result(myres); + + if (account_create_policy == "if-not-exists") { + // 2. Account doesn't exists, create new account. Otherwise reuse current + if (acc_exists == 0) { + pair new_creds { create_bootstrap_account(mysql, mon_user, mon_pass) }; + + if (new_creds.first) { + proxy_error("Bootstrap failed, user creation failed with error - %s\n", mysql_error(mysql)); + exit(EXIT_FAILURE); + } { + // Store the credentials as the new 'monitor' ones. + new_mon_user = mon_user; + new_mon_pass = new_creds.second.pass; + } + } else { + new_mon_user = mon_user; + new_mon_pass = mon_pass; + } + } else if (account_create_policy == "always") { + if (acc_exists == 0) { + pair new_creds { create_bootstrap_account(mysql, mon_user, mon_pass) }; + + if (new_creds.first) { + proxy_error("Bootstrap failed, user creation failed with error - %s\n", mysql_error(mysql)); + exit(EXIT_FAILURE); + } + } else { + proxy_error( + "Bootstrap failed, account '%s' already exists but supplied option '--account-create=\"always\"'\n", + mon_user.c_str() + ); + exit(EXIT_FAILURE); + } + + new_mon_user = mon_user; + new_mon_pass = mon_pass; + } else if (account_create_policy == "never") { + if (acc_exists == 0) { + proxy_error( + "Bootstrap failed, account '%s' doesn't exists but supplied option '--account-create=\"never\"'\n", + mon_user.c_str() + ); + exit(EXIT_FAILURE); + } + + new_mon_user = mon_user; + new_mon_pass = mon_pass; + } else { + proxy_error("Bootstrap failed, unknown '--account-create' option '%s'\n", account_create_policy.c_str()); + exit(EXIT_FAILURE); + } + } else { + string prev_bootstrap_user {}; + string prev_bootstrap_pass {}; + + if (Proxy_file_exists(GloVars.admindb)) { + SQLite3DB::LoadPlugin(GloVars.sqlite3_plugin); + SQLite3DB* configdb = new SQLite3DB(); + configdb->open((char*)GloVars.admindb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + + { + const char check_table[] { + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='bootstrap_variables'" + }; + + int table_exists = 0; + + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + configdb->execute_statement(check_table, &error , &cols , &affected_rows , &resultset); + + if (error == nullptr && resultset) { + table_exists = atoi(resultset->rows[0]->fields[0]); + delete resultset; + } else { + const char* err_msg = error != nullptr ? error : "Empty resultset"; + proxy_error("Bootstrap failed, query failed with error - %s", err_msg); + exit(EXIT_FAILURE); + } + + if (table_exists != 0) { + const char query_user_pass[] { + "SELECT variable_name,variable_value FROM bootstrap_variables" + " WHERE variable_name='bootstrap_username' OR variable_name='bootstrap_password'" + " ORDER BY variable_name" + }; + configdb->execute_statement(query_user_pass, &error, &cols, &affected_rows, &resultset); + + if (resultset->rows.size() != 0) { + prev_bootstrap_user = resultset->rows[1]->fields[1]; + prev_bootstrap_pass = resultset->rows[0]->fields[1]; + } + + if (resultset) { + delete resultset; + } + } + } + + delete configdb; + } + + if (!prev_bootstrap_pass.empty() && !prev_bootstrap_user.empty()) { + proxy_info( + "Bootstrap info, detected previous bootstrap user '%s' reusing account...\n", + prev_bootstrap_user.c_str() + ); + + new_mon_user = prev_bootstrap_user; + new_mon_pass = prev_bootstrap_pass; + } else { + // Create random account with random password + pair mon_creds { create_random_bootstrap_account(mysql, password_retries) }; + + if (mon_creds.first) { + proxy_error( + "Bootstrap failed, user creation '%s' failed with error - %s\n", + mon_creds.second.user.c_str(), mysql_error(mysql) + ); + exit(EXIT_FAILURE); + } else { + new_mon_user = mon_creds.second.user; + new_mon_pass = mon_creds.second.pass; + bootstrap_info.rand_gen_user = true; + } + } + } + + bootstrap_info.mon_user = new_mon_user; + bootstrap_info.mon_pass = new_mon_pass; + + mysql_close(mysql); + } + { cpu_timer t; ProxySQL_Main_init_SSL_module(); @@ -1497,7 +2144,7 @@ int main(int argc, const char * argv[]) { { cpu_timer t; - ProxySQL_Main_init_phase2___not_started(); + ProxySQL_Main_init_phase2___not_started(bootstrap_info); #ifdef DEBUG std::cerr << "Main init phase2 completed in "; #endif From 28d790a6e6d328d804ead0ddae6d431ac3f2762c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 29 May 2023 08:27:05 +0200 Subject: [PATCH 24/56] Fix minor GCC warnings --- lib/ProxySQL_Admin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 5fc7bdab8b..ff86e869b4 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1042,7 +1042,7 @@ bool FlushCommandWrapper(MySQL_Session *sess, const std::vector& cm else assert(0); msg += "\n"; - proxy_debug(PROXY_DEBUG_ADMIN, 4, msg.c_str()); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s", msg.c_str()); #endif // DEBUG SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return true; @@ -7936,7 +7936,7 @@ bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing( vector th_hashmaps {}; if (maps_per_thread) { - for (uint32_t i = 0; i < ths; i++) { + for (uint32_t i = 0; i < static_cast(ths); i++) { th_hashmaps.push_back(GloQPro->create_fast_routing_hashmap(resultset2)); } } From ff2e1a2b857c8e25ea95311f8248fa8cf6cb4744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 30 May 2023 13:52:54 +0200 Subject: [PATCH 25/56] Fix crash when attempting to log failed 'SET NAMES' Session 'status' and 'client_myds->DSS' should never be modified for a regular session prior to calling 'RequestEnd'. It's 'RequestEnd' responsibility to appropriately change these statuses after the rest of cleanup operations for the request has taken place, including logging. --- include/MySQL_Session.h | 18 +++++++++++++++--- lib/MySQL_Session.cpp | 4 ---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/include/MySQL_Session.h b/include/MySQL_Session.h index 424d7d6943..e0f569fd65 100644 --- a/include/MySQL_Session.h +++ b/include/MySQL_Session.h @@ -130,9 +130,21 @@ class MySQL_Session bool handler_CommitRollback(PtrSize_t *); bool handler_SetAutocommit(PtrSize_t *); /** - * @brief Performs the cleanup of current session state, and the required operations to the supplied - * 'MySQL_Data_Stream' required for processing further queries. - * @param The 'MySQL_Data_Stream' which executed the previous query and which status should be updated. + * @brief Should execute most of the commands executed when a request is finalized. + * @details Cleanup of current session state, and required operations to the supplied 'MySQL_Data_Stream' + * for further queries processing. Takes care of the following actions: + * - Update the status of the backend connection (if supplied), with previous query actions. + * - Log the query for the required statuses. + * - Cleanup the previous Query_Processor output. + * - Free the resources of the backend connection (if supplied). + * - Reset all the required session status flags. E.g: + * + status + * + client_myds::DSS + * + started_sending_data_to_client + * + previous_hostgroup + * NOTE: Should become the place to hook other functions. + * @param myds If not null, should point to a MySQL_Data_Stream (backend connection) which connection status + * should be updated, and previous query resources cleanup. */ void RequestEnd(MySQL_Data_Stream *); void LogQuery(MySQL_Data_Stream *); diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 55a331c009..790c167463 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -2480,8 +2480,6 @@ bool MySQL_Session::handler_again___status_CHANGING_CHARSET(int *_rc) { client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,mysql_errno(myconn->mysql),sqlstate,mysql_error(myconn->mysql)); myds->destroy_MySQL_Connection_From_Pool(true); myds->fd=0; - status=WAITING_CLIENT_DATA; - client_myds->DSS=STATE_SLEEP; RequestEnd(myds); } } else { @@ -7466,8 +7464,6 @@ void MySQL_Session::LogQuery(MySQL_Data_Stream *myds) { } } } -// this should execute most of the commands executed when a request is finalized -// this should become the place to hook other functions void MySQL_Session::RequestEnd(MySQL_Data_Stream *myds) { // check if multiplexing needs to be disabled char *qdt = NULL; From fe8428e1cf54c38cb9c1b80db56503b4154f7f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 30 May 2023 16:30:54 +0200 Subject: [PATCH 26/56] Remove/Move several explicit 'MySQL_Session' status flag changes Setting 'MySQL_Session::status', and 'client_myds::DSS' flags before a call to 'RequestEnd' is redundant, and in some scenarios even dangerous. 'RequestEnd' should handle this for regular sessions. --- lib/MySQL_Session.cpp | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 790c167463..0ac002332b 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -913,10 +913,11 @@ bool MySQL_Session::handler_CommitRollback(PtrSize_t *pkt) { uint16_t setStatus = 0; if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; if (mirror==false) { RequestEnd(NULL); + } else { + client_myds->DSS=STATE_SLEEP; + status=WAITING_CLIENT_DATA; } l_free(pkt->size,pkt->ptr); if (c=='c' || c=='C') { @@ -1059,10 +1060,11 @@ bool MySQL_Session::handler_SetAutocommit(PtrSize_t *pkt) { uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; if (mirror==false) { RequestEnd(NULL); + } else { + client_myds->DSS=STATE_SLEEP; + status=WAITING_CLIENT_DATA; } __sync_fetch_and_add(&MyHGM->status.autocommit_cnt_filtered, 1); } @@ -1286,10 +1288,11 @@ void MySQL_Session::return_proxysql_internal(PtrSize_t *pkt) { // default client_myds->DSS=STATE_QUERY_SENT_NET; client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,1064,(char *)"42000",(char *)"Unknown PROXYSQL INTERNAL command",true); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; if (mirror==false) { RequestEnd(NULL); + } else { + client_myds->DSS=STATE_SLEEP; + status=WAITING_CLIENT_DATA; } l_free(pkt->size,pkt->ptr); } @@ -1491,10 +1494,11 @@ bool MySQL_Session::handler_special_queries(PtrSize_t *pkt) { uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; if (mirror==false) { RequestEnd(NULL); + } else { + client_myds->DSS=STATE_SLEEP; + status=WAITING_CLIENT_DATA; } l_free(pkt->size,pkt->ptr); __sync_fetch_and_add(&MyHGM->status.frontend_set_names, 1); @@ -1508,10 +1512,11 @@ bool MySQL_Session::handler_special_queries(PtrSize_t *pkt) { resultset->add_column_definition(SQLITE_TEXT,"Message"); SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); delete resultset; - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; if (mirror==false) { RequestEnd(NULL); + } else { + client_myds->DSS=STATE_SLEEP; + status=WAITING_CLIENT_DATA; } l_free(pkt->size,pkt->ptr); return true; @@ -1521,10 +1526,11 @@ bool MySQL_Session::handler_special_queries(PtrSize_t *pkt) { if (mysql_thread___enable_load_data_local_infile == false) { client_myds->DSS=STATE_QUERY_SENT_NET; client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,1047,(char *)"HY000",(char *)"Unsupported 'LOAD DATA LOCAL INFILE' command",true); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; if (mirror==false) { RequestEnd(NULL); + } else { + client_myds->DSS=STATE_SLEEP; + status=WAITING_CLIENT_DATA; } l_free(pkt->size,pkt->ptr); return true; @@ -6515,8 +6521,6 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; RequestEnd(NULL); l_free(pkt->size,pkt->ptr); return true; @@ -6558,8 +6562,6 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; RequestEnd(NULL); l_free(pkt->size,pkt->ptr); return true; @@ -6598,8 +6600,6 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; RequestEnd(NULL); l_free(pkt->size,pkt->ptr); return true; @@ -7647,8 +7647,6 @@ bool MySQL_Session::handle_command_query_kill(PtrSize_t *pkt) { uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); if (autocommit) setStatus = SERVER_STATUS_AUTOCOMMIT; client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; RequestEnd(NULL); l_free(pkt->size,pkt->ptr); return true; From f8d5fedc538c26590edecf338cc80a8ae8d476d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 30 May 2023 18:21:26 +0200 Subject: [PATCH 27/56] Improve documentation for '--bootstrap' command line option --- lib/ProxySQL_GloVars.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index 15c8c9de39..64f94a1ec2 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -253,7 +253,9 @@ ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : opt->add((const char *)"",0,0,0,(const char *)"Enable SQLite3 Server",(const char *)"--sqlite3-server"); // Bootstrap General options - opt->add((const char *)"",0,1,0,(const char *)"Start ProxySQL in Group Replication bootstrap mode",(const char *)"-B", (const char *)"--bootstrap"); + opt->add((const char *)"",0,1,0,(const char *)"Start ProxySQL in Group Replication bootstrap mode." + " An URI needs to be specified for creating a connection to the bootstrap server, if no URI is provided," + " a connection to the default local socket will be attempted.",(const char *)"-B", (const char *)"--bootstrap"); opt->add((const char *)"",0,1,0, (const char *)"Account to use by monitoring after bootstrap, either reuses a specify account or creates a new one;" " this behavior is controlled by related option '--acount-create'. When used, a password must be provided." ,(const char *)"--account"); opt->add((const char *)"",0,1,0,(const char *)"Account creation policy for bootstrap. Possible values are:\n" From a999d621864bd174150dafc55d252b074e1c1830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 30 May 2023 19:26:21 +0200 Subject: [PATCH 28/56] Fix typo on 'BOOTSTRAP_SELECT_USERS' query for RELEASE builds --- src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index e1f9b558de..c6cf57d2ad 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1504,7 +1504,7 @@ const char BOOTSTRAP_SELECT_USERS[] { "SELECT DISTINCT user,ssl_type,authentication_string,plugin,password_expired FROM mysql.user" " WHERE user NOT LIKE 'mysql.%' AND user NOT LIKE 'bt_proxysql_%'" #ifndef DEBUG - " AND user != 'root'"; + " AND user != 'root'" #endif }; From ae911d5f23cab93d253d9015ac53230a45a0b564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 31 May 2023 12:12:06 +0200 Subject: [PATCH 29/56] Improve parsing of URI used for bootstrap mode - URI parsing now handles properly more optional groups. - Fixed default port when optional group is missing from URI. - Some URI sensitive info has been excluded from non-debug builds. --- src/main.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index c6cf57d2ad..80263125f1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1355,18 +1355,25 @@ namespace { * - '127.0.0.1:3306' * - 'mysql-server-1:3306' * + UserInfo is inside a non-capturing group to avoid matching '@'. + * + Host,Port groups are inside a non-capturing group to allow URI like: 'mysql://user:pass@' * + RegName matches any valid Ipv4 or domain name. * + Ipv6 matches Ipv6, it's NOT spec conforming, we don't verify the supplied Ip in the regex. * + Port matches the supplied port. + * + Optionally match a supplied (/). + * + Ensure match termination in HierPart group, forcing conditional subgroups matching. */ const char CONN_URI_REGEX[] { "^(:?(?P[a-z][a-z0-9\\+\\-\\.]*):\\/\\/)?" - "(?P" - "(?:(?P(?:\\%[0-9a-f][0-9a-f]|[a-z0-9\\-\\.\\_\\~]|[\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\=]|\\:)*)\\@)?" + "(?P" + "(?:(?P(?:\\%[0-9a-f][0-9a-f]|[a-z0-9\\-\\.\\_\\~]|[\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\=]|\\:)*)\\@)?" + "(:?" "(?P" "(?P(?:\\%[0-9a-f][0-9a-f]|[a-z0-9\\-\\.\\_\\~]|[\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\=]])*)|" - "(?P\\[(?:[0-9a-f]|[\\:])*\\])):" - "(?P[0-9]+))?" + "(?P\\[(?:[0-9a-f]|[\\:])*\\]))" + "(?:\\:(?P[0-9]+)?)?" + ")?" + "(?:\\/)?" + ")?$" }; /** @@ -1483,7 +1490,9 @@ string to_string(const conn_uri_t& conn_uri) { j["scheme"] = conn_uri.scheme; j["user"] = conn_uri.user; +#ifdef DEBUG j["pass"] = conn_uri.pass; +#endif j["host"] = conn_uri.host; j["port"] = conn_uri.port; @@ -1750,15 +1759,18 @@ int main(int argc, const char * argv[]) { proxy_info("Aborting bootstrap due to failed to parse or match URI - `%s`\n", to_string(uri_data).c_str()); exit(parse_uri_res.first); } else { - proxy_info("Starting bootstrap connection with data extracted from URI - `%s`\n", to_string(uri_data).c_str()); + proxy_info("Bootstrap connection data supplied via URI - `%s`\n", to_string(uri_data).c_str()); } const char* c_host = uri_data.host.c_str(); const char* c_user = uri_data.user.empty() ? "root" : uri_data.user.c_str(); const char* c_pass = nullptr; - uint32_t port = uri_data.port; + uint32_t port = uri_data.port == 0 ? 3306 : uri_data.port; uint32_t flags = CLIENT_SSL; + nlohmann::ordered_json conn_data { { "host", c_host }, { "user", c_user }, { "port", port } }; + proxy_info("Performing bootstrap connection using URI data and defaults - `%s`\n", conn_data.dump().c_str()); + if (uri_data.pass.empty()) { c_pass = getpass("Enter password: "); } else { From f5a548c2740fadb930b02a82145251c5399d371e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 31 May 2023 13:07:58 +0200 Subject: [PATCH 30/56] Fix compilation for Centos 6 --- lib/ProxySQL_Admin.cpp | 4 ++-- src/main.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 6d56ac7b69..43ac133723 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -6089,8 +6089,8 @@ string build_boot_servers_insert(const vector& srvs_info_defs) { string t_srvs_values {}; for (const auto& info_defs : srvs_info_defs) { - const boot_srv_info_t& srv_info { info_defs.first }; - const srv_defs_t& srv_defs { info_defs.second }; + const boot_srv_info_t& srv_info = info_defs.first; + const srv_defs_t& srv_defs = info_defs.second; const char t_values[] { "(%d, \"%s\", %d, \"%s\", %ld, %ld, %d)" }; string srv_values = cstr_format( diff --git a/src/main.cpp b/src/main.cpp index 80263125f1..47ef809a4f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1753,7 +1753,7 @@ int main(int argc, const char * argv[]) { // - Connection string parsing is required const string conn_uri { GloVars.global.gr_bootstrap_uri }; const pair parse_uri_res { parse_conn_uri(conn_uri) }; - const conn_uri_t uri_data { parse_uri_res.second }; + const conn_uri_t uri_data = parse_uri_res.second; if (parse_uri_res.first == EXIT_FAILURE) { proxy_info("Aborting bootstrap due to failed to parse or match URI - `%s`\n", to_string(uri_data).c_str()); From 249f023fbea14b7caefd1834f7af3ed0e675641b Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Mon, 26 Jun 2023 02:35:49 +0500 Subject: [PATCH 31/56] Added ProxySQL Cluster internal working documentation and flowchart TODO: Update documentation: mysql_servers_v2 --- .../proxysql_cluster_working.md | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 doc/proxysql_cluster/proxysql_cluster_working.md diff --git a/doc/proxysql_cluster/proxysql_cluster_working.md b/doc/proxysql_cluster/proxysql_cluster_working.md new file mode 100644 index 0000000000..bb91a5089b --- /dev/null +++ b/doc/proxysql_cluster/proxysql_cluster_working.md @@ -0,0 +1,222 @@ +# Introduction +This documentation provides an in-depth look at the internal workings of the ProxySQL Cluster feature. It is intended for readers who are already familiar with the basic concepts and functionality of ProxySQL Cluster. + +# Prerequisites +Before reading this documentation, it is mandatory that the reader has gone through the official ProxySQL Cluster documentation available at [https://proxysql.com/documentation/proxysql-cluster/](https://proxysql.com/documentation/proxysql-cluster/). This will provide the necessary background knowledge and understanding of terminologies to understand the internal workings of the feature. + +# Important classes used by ProxySQL Cluster +This section describes the classes that are used by the ProxySQL Cluster feature. + +## ProxySQL_Cluster +The `ProxySQL_Cluster` class is the core component for managing ProxySQL clusters. It provides a wide range of operations and functionalities to handle and manage the cluster effectively. With `ProxySQL_Cluster`, you can add or remove nodes, manage node weights, monitor cluster status, and perform other essential tasks. + +## ProxySQL_Node_Entry +The `ProxySQL_Node_Entry` class represents an individual node within a ProxySQL cluster. It serves as a container for storing information and provides convenient methods to access node-specific details. It consists of important node attributes such as its status, weight, connection statistics, and contains global_checksum and checksum, epoch and version of individual modules of that node. + +## ProxySQL_Cluster_Nodes +The `ProxySQL_Cluster_Nodes` class serves as a centralized manager for all the nodes within a ProxySQL cluster. It provides a cohesive interface to handle operations related to the entire cluster's nodes efficiently. Through the `ProxySQL_Cluster_Nodes` class, you can easily manage and manipulate various node-specific actions within the cluster context. It consists of an unordered map that contains all the nodes. + +## ProxySQL_Node_Address +The `ProxySQL_Node_Address` class represents a ProxySQL node's address and encapsulates the hostname, port, node UUID, admin_mysql_ifaces, and optional IP address. + +## ProxySQL_GlobalVariables +The `ProxySQL_GlobalVariables` class contains checksum, epoch, and version of individual modules of the current ProxySQL instance. It provides a way to access and manage the global variables related to the ProxySQL Cluster. + +## ProxySQL_Checksum_Value +The `ProxySQL_Checksum_Value` class is used to store checksum values for modules. It includes member variables such as version and epoch to keep track of the version and epoch of the checksum value. + + +# Initializing Monitoring Threads +Each node within the ProxySQL cluster monitors core nodes to identify any updates in module configuration. This is achieved by dedicating a separate thread for each core node, for continuous monitoring for changes. + +## Load ProxySQL Servers +When the ProxySQL instance is started or the `LOAD PROXYSQL SERVERS TO RUNTIME` command is executed, the latest `proxysql_servers` table records are fetched from the database. This result set is passed to the ProxySQL Cluster's `load_servers_list` method. + +## Initialize ProxySQL Cluster Nodes +The retrieved result set is used to initialize the ProxySQL Cluster Nodes. The `load_servers_list` method of `ProxySQL_Cluster_Nodes` creates a map of all the nodes by utilizing the `ProxySQL_Node_Entry` objects. Each `ProxySQL_Node_Entry` object represents a node in the cluster and holds information such as the host, port, weight, and comment of the node. + +## Mark all nodes Inactive +Mark all the nodes in the `ProxySQL_Cluster_Nodes` map as inactive. + +## Check Node Existence and Update +For each node in the retrieved result set, the existence of the node is checked within the `ProxySQL_Cluster_Nodes` map. If the node already exists in the map, it is marked as active, and any changes in the weights and comments are updated accordingly. + +## Create New Node Entry +If a node does not exist in the map, a new `ProxySQL_Node_Entry` object is created for that node. The `ProxySQL_Node_Entry` is initialized with the host, port, weight, and comment of the node. This new entry is then inserted into the ProxySQL Cluster Nodes map. + +## Create Monitoring Thread +Subsequently, a monitoring thread is created for each node in the ProxySQL cluster. These monitoring threads are responsible for effectively managing and monitoring the status and performance of each node. The `ProxySQL_Node_Address`, containing the host and port of the node, is passed as an argument to the monitoring thread. + +## Remove Inactive Nodes +After iterating through all the entries in the result set and performing the necessary operations, any inactive nodes present in the `ProxySQL_Cluster_Nodes` map are deleted. This ensures that only active and relevant nodes remain within the cluster. + +## Flowchart: +```mermaid +flowchart TD + +subgraph Cluster Nodes Monitoring Threads + start[Start] --> load[LOAD PROXYSQL SERVERS TO RUNTIME] --> select[SELECT hostname, port, weight, comment FROM proxysql_servers ORDER BY hostname, port] + select -- ProxySQL_Cluster::load_servers_list -> ProxySQL_Cluster_Nodes::load_servers_list ->ProxySQL_Cluster_Nodes::set_all_inactive --> test + test[Set all nodes to inactive state] + test -- ProxySQL_Cluster::load_servers_list -> ProxySQL_Cluster_Nodes::load_servers_list --> check[Next node available in resultset?] + check -- NO --> preend[PREEND] + check -- YES
ProxySQL_Cluster_Nodes::umap_proxy_nodes --> check_map{Node exists in map?} + check_map -- YES --> update[Update node weight, comment, and set node state to active] + check_map -- NO --> create1[Create new ProxySQL_Node_Entry with node host, port, weight, and comment and set node state to active and insert it in the map
Create new thread for node and send ProxySQL_Node_Address with node host and port as argument to that thread] --> check + update --> check + preend[Remove all inactive nodes] --> END[END] + END[END] +end +``` + +# Monitoring Thread Working +This section describes how each monitoring thread representing a specific ProxySQL cluster node performs its operations. + +## Check ProxySQL Version +The thread compares the ProxySQL version of the node with the local instance's ProxySQL version by executing the `SELECT @@version` query. If the versions do not match, the connection with the remote peer is terminated. + +## Register Node +If the versions match, `PROXYSQL CLUSTER_NODE_UUID ProxySQL_GlobalVariables::uuid ProxySQL_Cluster::admin_mysql_ifaces` command is sent to the remote peer to register current node as a client. + +## Global Checksum +Global checksum is fetched from the remote peer by sending `SELECT GLOBAL_CHECKSUM()` query. The resultset is passed to the `ProxySQL_Cluster::Update_Global_Checksum` method, which finds the corresponding `ProxySQL_Node_Entry` in the `ProxySQL_Cluster_Nodes` map. If the new global checksum is different from the previously saved one, the global checksum is updated in the `ProxySQL_Node_Entry`. + +## Module Checksum +If the global checksum is updated, indicating changes in one or more modules. `SELECT * FROM runtime_checksums_values ORDER BY name` query is sent to the remote peer to fetch the latest checksums of all modules, including epoch and version. + +## Compare and Update Checksums +The resultset of the query is passed to the `ProxySQL_Cluster::Update_Node_Checksums` method, which finds the corresponding `ProxySQL_Node_Entry` in the `ProxySQL_Cluster_Nodes` map. The method then compares the checksum values of each module with the recently fetched resultset. If a checksum value is different, it is replaced with the latest one in the `ProxySQL_Node_Entry`, and the diff_check of that module is incremented. + +## Sync Configuration +For each module that has a `ProxySQL_Cluster::cluster_*module_name*_diffs_before_sync` value greater than zero, indicating that the module is enabled for syncing, the thread proceeds to check the node's version and epoch. If the node's version is greater than 1 and own_version is equal to 1 (means instance is just booted) or node's epoch is greater than own_epoch, then it checks diff_check. If diff_check is greater than `ProxySQL_Cluster::cluster_*module_name*_diffs_before_sync`, it fetches latest configuration of that module using `ProxySQL_Cluster::pull_*module_name*_from_peer` method. + +## Select Sync Source +The thread calls `ProxySQL_Cluster_Nodes::get_peer_to_sync_*module_name*` to iterate over `ProxySQL_Cluster_Nodes` map and find a node with a version greater than 1 and maximum epoch value among all nodes. This node is selected as source of truth for syncing. + +## Fetch and Apply Configuration +Connection is created to selected node using credentials obtained from `ProxySQL_Cluster::get_credentials` and fetches latest configuration for the module. After fetching configuration, it computes checksum locally and compares it with checksum received from node. If checksums match, changes are applied to runtime. Otherwise changes are discarded. + +## Save Configuration +If "save to disk" variable is set to true, configuration of that module is saved to disk ensuring persistence. + +## Flowchart (Simplified): +```mermaid + graph TB + A[Start] --> A1 + A1["Connect to remote peer and fetch ProxySQL version"] --> B + B{Compare ProxySQL version of remote peer with local instance} + B -->|Versions are not same| C[Close connection with the remote peer] + B -->|Versions are same| D[Register current node as a client] + D --> E[Fetch global checksum from remote peer] + E --> F{Compare global checksum with local global checksum} + F -->|Checksums are same| G[Do nothing] + F -->|Checksums are different| H[Override previous local global checksum with new one] + H --> K[Fetch checksums of all modules from peer node] + K --> N{Compare local module checksum_values with fetched remote peer resultset} + N -->|Checksums are same| O[Do nothing] + N -->|Checksums are different| P[Replace checksum and increment module diff_check] + P --> Q{Check if module cluster_*module_name*_diffs_before_sync > 0} + Q -->|Value is not greater than 0| R[Do nothing] + Q -->|Value is greater than 0| S{Own_version == 1 or local epoch > own_epoch?} + S -->|False| T[Do nothing] + S -->|True| U{diff_check >= cluster_*module_name*_diffs_before_sync?} + U -->|False| V[Do nothing] + U -->|True| X[Find node with version > 1 and max_epoch value] + X --> Y[Connect and fetch module's latest configuration from thet remote peer] + Y --> Y1[Compute checksum of fetched module configuration] + Y1 --> Z{Compare locally computed checksum with remote peer checksum} + Z -->|Checksums are not same| AA[Discard changes] + Z -->|Checksums are same| AB["Apply configuration changes to runtime"] + AB --> AC{Check cluster_*module_name*_save_to_disk == true?} + AC -->|False| AD[Do nothing] + AC -->|True| AE[Save configuration of the module to disk] + AE --> AF[End] +``` + +## Flowchart (Detailed): +```mermaid +graph TD + subgraph ProxySQL_Cluster_Monitor_thread[ProxySQL_Cluster_Monitor_thread] + direction TB + A[Start] -- "ProxySQL_Cluster_Monitor_thread
ProxySQL_Node_Address -> host, port" --> B + B[Connect to remote peer using host and port] --> C + C["Send `SELECT @@version`"] -- "resultset contains remote peer ProxySQL version" --> D + D{Remote peer ProxySQL version == Local PROXYSQL_VERSION?} + D -- "No" --> CLOSE_CONNECTION + D -- "Yes
Register yourself with a remote peer" --> E + E["Send `PROXYSQL CLUSTER_NODE_UUID ProxySQL_GlobalVariables::uuid ProxySQL_Cluster::admin_mysql_ifaces`"] --> F + F{ProxySQL_GlobalVariables::shutdown == 0} + F -- "Yes
Fetch Global Checksum from remote peer" --> G + F -- "No" --> CLOSE_CONNECTION + G["Send `SELECT GLOBAL_CHECKSUM()`"] -- "resultset contains global checksum of remote peer
ProxySQL_Cluster::Update_Global_Checksum" --> I + I{Local global checksum == Remote peer global checksum?} + I -- "Yes" --> AAA + I -- "No" --> J + J[Update local global checksum with remote peer global checksum value] -- "return checksum_updated = true
ProxySQL_Cluster_Monitor_thread

one or more module configuration had been changed. Fetching all the modules checksum, version and epoch from remote peer" --> K + K["Send `SELECT * FROM runtime_checksums_values ORDER BY name`"] -- "resultset contains module name, checksum, version and epoch
ProxySQL_Cluster_Nodes::Update_Node_Checksums
ProxySQL_Node_Entry::set_checksums" --> M + subgraph set_checksum[ProxySQL_Node_Entry::set_checksums] + direction TB + M[Load cluster_*module_name*_diffs_before_sync variables] --> N + N{resultset != NULL && record availabe in resultset?} -- "Yes

**logic is same for all module: module name == admin_variables
module name == mysql_query_rules
module name == mysql_servers
module name == mysql_users
module name == mysql_variables
module name == proxysql_servers
module name == ldap_variables**" --> O + O["Update local module version, epoch and last_updated value with the remote peer value"] --> P + P{Local module checksum == Remote peer module checksum?} + P -- "Yes" --> Q + P -- "No" --> S + Q[Set local module diff_check += 1] --> T + S[Update local module checksum with remote peer checksum value, last_changed to current time and diff_check = 1] --> T + T{Own module checksum == Local module checksum?} + T -- "Yes" --> U + T -- "No" --> N + U[Set local module diff_check = 0] --> N + N -- "No
cluster_*module_name*_diffs_before_sync variables" --> R + R{resultset == NULL?} + R -- "Yes
For every module" --> YY + YY[Set local module last_updated = current time] --> XX + XX{Local module checksum == Own module version?} + XX -- "Yes" --> WW + WW[Set local module diff_check = 0] --> V + XX -- "No" --> VV + VV[Set local module diff_check += 1] --> V + V{cluster_*module_name*_diffs_before_sync variables != 0?} + V -- "Yes" --> W + V -- "No" --> DELAY + W{Local module version > 1?} + W -- "Yes" --> X + W -- "No" --> V + X{"Own module version == 1 || Local module epoch > Own module epoch?"} + X -- "Yes" --> Y + X -- "No" --> V + Y{Local module diff_check >= cluster_*module_name*_diffs_before_sync?} + Y -- "No" --> V + Y -- "Yes
ProxySQL_Cluster_Nodes::umap_proxy_nodes" --> Y1 + Y1["Find local node with version > 1 and max_epoch"] -- "ProxySQL_Cluster::pull_*module_name*_from_peer" --> Z + Z["Connect to that remote peer and fetch latest module configuration"] --> AA + AA["Locally compute checksum of fetched configuration"] --> BB + BB{"Locally computed checksum == Remote peer checksum?"} + BB -- "Yes" --> DD + BB -- "No" --> DELAY + DD["Delete records in local module configuration table(s)"] --> EE + EE["Insert retrieved data from remote peer into local module configuration table(s)"] --> FF + FF["Issue internal `LOAD *module_name* TO RUNTIME`"] --> GG + GG{"Check cluster_*module_name*_save_to_disk == true?"} + GG -- "Yes" --> II + GG -- "No" --> AAA + II[Issue internal `SAVE *module_name* TO DISK`] + II --> AAA + end + AAA{"Local cluster_check_status_frequency_count >= cluster_check_status_frequency?"} + AAA -- "No" --> BBB + AAA -- Yes --> DDD + BBB[Set cluster_check_status_frequency_count += 1] --> DELAY + DDD[Set cluster_check_status_frequency_count = 0] --> EEE + EEE[Send `SELECT * FROM stats_mysql_global ORDER BY Variable_Name`] + EEE -- "resultset contains Client_Connections_connected, Client_Connections_created, ProxySQL_Uptime, Questions, Servers_table_version of remote peer
ProxySQL_Cluster_Nodes::Update_Node_Metrics
ProxySQL_Node_Entry::set_metrics" --> FFF + subgraph set_metrics[ProxySQL_Node_Entry::set_metrics] + direction TB + FFF[Update local metrices with values from resultset] --> DELAY + end + DELAY[Sleep cluster_check_interval_ms] --> F + CLOSE_CONNECTION[Close Connection] --> END + END[End] +end +``` \ No newline at end of file From bf5d8cbc73f2ba1044f248be8f8a1bc11098996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 28 Jun 2023 16:35:49 +0200 Subject: [PATCH 32/56] Add functions for breaking down 'commit' checksum generation --- include/MySQL_HostGroups_Manager.h | 3 + include/proxysql_glovars.hpp | 14 +-- include/proxysql_utils.h | 36 ++++++ lib/MySQL_HostGroups_Manager.cpp | 171 +++++++++++++++++++++++++++++ lib/ProxySQL_Cluster.cpp | 11 -- lib/proxysql_utils.cpp | 22 ++++ 6 files changed, 234 insertions(+), 23 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index e370e926d8..239af7be6f 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -759,7 +759,10 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); + std::string gen_global_mysql_servers_checksum(); bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0); + void commit_generate_mysql_servers_table(SQLite3_result* runtime_mysql_servers = nullptr); + void commit_update_checksum_from_mysql_servers(SpookyHash& myhash, bool& init); void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init); void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() diff --git a/include/proxysql_glovars.hpp b/include/proxysql_glovars.hpp index fe23e756fd..e4ff075fe8 100644 --- a/include/proxysql_glovars.hpp +++ b/include/proxysql_glovars.hpp @@ -5,27 +5,17 @@ #define CLUSTER_SYNC_INTERFACES_MYSQL "('mysql-interfaces')" #include +#include #include #include "configfile.hpp" #include "proxy_defines.h" +#include "proxysql_utils.h" namespace ez { class ezOptionParser; }; -/** - * @brief Helper function used to replace spaces and zeros by '0' char in the supplied checksum buffer. - * @param checksum Input buffer containing the checksum. - */ -inline void replace_checksum_zeros(char* checksum) { - for (int i=2; i<18; i++) { - if (checksum[i]==' ' || checksum[i]==0) { - checksum[i]='0'; - } - } -} - #ifndef ProxySQL_Checksum_Value_LENGTH #define ProxySQL_Checksum_Value_LENGTH 20 #endif diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 76a1c2e302..00952efd9e 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -2,12 +2,19 @@ #define __PROXYSQL_UTILS_H #include +#include #include #include #include #include #include +#include "sqlite3db.h" + +#ifndef ProxySQL_Checksum_Value_LENGTH +#define ProxySQL_Checksum_Value_LENGTH 20 +#endif + #ifndef ETIME // ETIME is not defined on FreeBSD // ETIME is used internaly to report API timer expired @@ -212,4 +219,33 @@ std::string replace_str(const std::string& str, const std::string& match, const std::vector split_str(const std::string& s, char delimiter); std::string generate_multi_rows_query(int rows, int params); + +/** + * @brief Helper function used to replace spaces and zeros by '0' char in the supplied checksum buffer. + * @param checksum Input buffer containing the checksum. + */ +inline void replace_checksum_zeros(char* checksum) { + for (int i=2; i<18; i++) { + if (checksum[i]==' ' || checksum[i]==0) { + checksum[i]='0'; + } + } +} + +/** + * @brief Generates a ProxySQL checksum as a string from the supplied integer hash. + * @param hash The integer hash to be formated as a string. + * @return String representation of the supplied hash. + */ +std::string get_checksum_from_hash(uint64_t hash); + +/** + * @brief Remove the rows from the resultset matching the supplied predicate. + * @param resultset The resultset which rows are to be removed. + * @param pred Predicate that should return 'true' for the rows to be removed. + */ +void remove_sqlite3_resultset_rows( + std::unique_ptr& resultset, const std::function& pred +); + #endif diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 1dc975d313..d6de5d1fb6 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1621,6 +1621,177 @@ void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& m CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); } +const char MYSQL_SERVERS_CHECKSUM_QUERY[] { + "SELECT hostgroup_id, hostname, port, gtid_port, CASE WHEN status=0 OR status=1 OR status=4 THEN 0 ELSE status END status," + " weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers" + " WHERE status<>3 ORDER BY hostgroup_id, hostname, port" +}; + +/** + * @brief Generates a resultset which is used to compute the current 'mysql_servers' checksum. + * @details The resultset should report all servers status as ONLINE(0), with the exception of 'OFFLINE_HARD'. + * Servers with this status should be excluded from the resultset. + * @param mydb The db in which to perform the query, typically 'MySQL_HostGroups_Manager::mydb'. + * @return An SQLite3 resultset for the query 'MYSQL_SERVERS_CHECKSUM_QUERY'. + */ +unique_ptr get_mysql_servers_checksum_resultset(SQLite3DB* mydb) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + mydb->execute_statement(MYSQL_SERVERS_CHECKSUM_QUERY, &error, &cols, &affected_rows, &resultset); + + if (error) { + proxy_error("Checksum generation query for 'mysql_servers' failed with error '%s'\n", error); + assert(0); + } + + return unique_ptr(resultset); +} + +/** + * @brief Generates a resultset holding the current Admin 'runtime_mysql_servers' as reported by Admin. + * @param mydb The db in which to perform the query, typically 'MySQL_HostGroups_Manager::mydb'. + * @return An SQLite3 resultset for the query 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. + */ +unique_ptr get_admin_runtime_mysql_servers(SQLite3DB* mydb) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + return unique_ptr(resultset); +} + +/** + * @brief Removes rows with 'OFFLINE_HARD' servers in the supplied resultset. + * @details It assumes that the supplied resultset is generated via 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. + * @param resultset The resultset from which rows are to be removed. + */ +void remove_resultset_offline_hard_servers(unique_ptr& resultset) { + if (resultset->columns < 5) { + return; + } + + const auto is_offline = [] (SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + return true; + } else { + return false; + } + }; + + remove_sqlite3_resultset_rows(resultset, is_offline); +} + +/** + * @brief Updates the global 'mysql_servers' checksum. + * @details If the new computed checksum matches the supplied 'cluster_checksum', the epoch used for the + * checksum, is the supplied epoch instead of current time. This way we ensure the preservation of the + * checksum and epoch fetched from the ProxySQL cluster peer node. + * + * @param new_checksum The new computed checksum by ProxySQL. + * @param old_checksum A checksum, previously fetched from ProxySQL cluster. Should be left empty if the + * update isn't considering this scenario. + * @param epoch The epoch to be preserved in case the supplied 'cluster_checksum' matches the new computed + * checksum. + */ +void update_glovars_mysql_servers_checksum( + const std::string& new_checksum, const std::string& cluster_checksum = "", const time_t epoch = 0 +) { + GloVars.checksums_values.mysql_servers.set_checksum(const_cast(new_checksum.c_str())); + GloVars.checksums_values.mysql_servers.version++; + + time_t t = time(NULL); + bool computed_checksum_matches { + cluster_checksum != "" && GloVars.checksums_values.mysql_servers.checksum == cluster_checksum + }; + + if (epoch != 0 && computed_checksum_matches) { + GloVars.checksums_values.mysql_servers.epoch = epoch; + } else { + GloVars.checksums_values.mysql_servers.epoch = t; + } + + GloVars.checksums_values.updates_cnt++; + GloVars.generate_global_checksum(); + GloVars.epoch_version = t; +} + +void MySQL_HostGroups_Manager::commit_generate_mysql_servers_table(SQLite3_result* runtime_mysql_servers) { + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + if (runtime_mysql_servers == nullptr) { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + + // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster nodes, or + // relevant for checksum computation. If this step isn't performed, this could cause mismatching + // checksums between different primary nodes in a ProxySQL cluster, since OFFLINE_HARD servers + // preservation depends on unknown and unpredictable connections conditions. + remove_resultset_offline_hard_servers(resultset); + save_runtime_mysql_servers(resultset.release()); + } else { + save_runtime_mysql_servers(runtime_mysql_servers); + } +} + +void MySQL_HostGroups_Manager::commit_update_checksum_from_mysql_servers(SpookyHash& myhash, bool& init) { + unique_ptr mysrvs_checksum_resultset { get_mysql_servers_checksum_resultset(mydb) }; + + // Reset table checksum value before recomputing + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = 0; + + if (mysrvs_checksum_resultset) { + if (mysrvs_checksum_resultset->rows_count) { + if (init == false) { + init = true; + myhash.Init(19,3); + } + uint64_t hash1_ = mysrvs_checksum_resultset->raw_checksum(); + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; + + myhash.Update(&hash1_, sizeof(hash1_)); + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); + } + } else { + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); + } +} + +std::string MySQL_HostGroups_Manager::gen_global_mysql_servers_checksum() { + SpookyHash global_hash; + bool init = false; + + // Regenerate 'mysql_servers' and generate new checksum, initialize the new 'global_hash' + commit_update_checksum_from_mysql_servers(global_hash, init); + + // Complete the hash with the rest of the unchanged modules + for (size_t i = 0; i < table_resultset_checksum.size(); i++) { + uint64_t hash_val = table_resultset_checksum[i]; + + if (i != HGM_TABLES::MYSQL_SERVERS && hash_val != 0) { + if (init == false) { + init = true; + global_hash.Init(19, 3); + } + + global_hash.Update(&hash_val, sizeof(hash_val)); + } + } + + uint64_t hash_1 = 0, hash_2 = 0; + if (init) { + global_hash.Final(&hash_1,&hash_2); + } + + string mysrvs_checksum { get_checksum_from_hash(hash_1) }; + return mysrvs_checksum; +} + bool MySQL_HostGroups_Manager::commit( SQLite3_result* runtime_mysql_servers, const std::string& checksum, const time_t epoch ) { diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 6d9d42f9f0..63d39983d6 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -967,17 +967,6 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } } -std::string get_checksum_from_hash(uint64_t hash) { - uint32_t d32[2] = { 0 }; - memcpy(&d32, &hash, sizeof(hash)); - - vector s_buf(20, 0); - sprintf(&s_buf[0],"0x%0X%0X", d32[0], d32[1]); - replace_checksum_zeros(&s_buf[0]); - - return string { &s_buf.front() }; -} - /** * @brief Computes the checksum from a MySQL resultset in the same we already do in 'SQLite3_result::raw_checksum'. * @details For each received column computing the field length via 'strlen' is required, this is because we diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index 2c20fcc7d4..4d72fd49e1 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -1,5 +1,6 @@ #include "proxysql_utils.h" +#include #include #include @@ -12,7 +13,9 @@ #include #include +using std::function; using std::string; +using std::unique_ptr; using std::vector; __attribute__((__format__ (__printf__, 1, 2))) @@ -409,3 +412,22 @@ std::string generate_multi_rows_query(int rows, int params) { } return s; } + +std::string get_checksum_from_hash(uint64_t hash) { + uint32_t d32[2] = { 0 }; + memcpy(&d32, &hash, sizeof(hash)); + + vector s_buf(ProxySQL_Checksum_Value_LENGTH, 0); + sprintf(&s_buf[0],"0x%0X%0X", d32[0], d32[1]); + replace_checksum_zeros(&s_buf[0]); + + return string { &s_buf.front() }; +} + +void remove_sqlite3_resultset_rows( + unique_ptr& resultset, const function& pred +) { + const auto remove_it { std::remove_if(resultset->rows.begin(), resultset->rows.end(), pred) }; + resultset->rows.erase(remove_it, resultset->rows.end()); + resultset->rows_count = resultset->rows.size(); +} From 4f73331defd1f6ca0d8e7faafe92d23a96c70cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 28 Jun 2023 16:35:49 +0200 Subject: [PATCH 33/56] Add functions for breaking down 'commit' checksum generation --- include/MySQL_HostGroups_Manager.h | 3 + include/proxysql_glovars.hpp | 14 +-- include/proxysql_utils.h | 35 ++++++ lib/MySQL_HostGroups_Manager.cpp | 171 +++++++++++++++++++++++++++++ lib/ProxySQL_Cluster.cpp | 11 -- lib/proxysql_utils.cpp | 21 ++++ 6 files changed, 232 insertions(+), 23 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 1d32b6a3e5..b7549b3cf9 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -759,7 +759,10 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); + std::string gen_global_mysql_servers_checksum(); bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0); + void commit_generate_mysql_servers_table(SQLite3_result* runtime_mysql_servers = nullptr); + void commit_update_checksum_from_mysql_servers(SpookyHash& myhash, bool& init); void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init); void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() diff --git a/include/proxysql_glovars.hpp b/include/proxysql_glovars.hpp index e2eb14c8af..0653f10d0d 100644 --- a/include/proxysql_glovars.hpp +++ b/include/proxysql_glovars.hpp @@ -5,27 +5,17 @@ #define CLUSTER_SYNC_INTERFACES_MYSQL "('mysql-interfaces')" #include +#include #include #include "configfile.hpp" #include "proxy_defines.h" +#include "proxysql_utils.h" namespace ez { class ezOptionParser; }; -/** - * @brief Helper function used to replace spaces and zeros by '0' char in the supplied checksum buffer. - * @param checksum Input buffer containing the checksum. - */ -inline void replace_checksum_zeros(char* checksum) { - for (int i=2; i<18; i++) { - if (checksum[i]==' ' || checksum[i]==0) { - checksum[i]='0'; - } - } -} - #ifndef ProxySQL_Checksum_Value_LENGTH #define ProxySQL_Checksum_Value_LENGTH 20 #endif diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 5ccbb4eb7c..3660bb0a68 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -2,12 +2,19 @@ #define __PROXYSQL_UTILS_H #include +#include #include #include #include #include #include +#include "sqlite3db.h" + +#ifndef ProxySQL_Checksum_Value_LENGTH +#define ProxySQL_Checksum_Value_LENGTH 20 +#endif + #ifndef ETIME // ETIME is not defined on FreeBSD // ETIME is used internaly to report API timer expired @@ -220,4 +227,32 @@ std::string generate_multi_rows_query(int rows, int params); */ std::string rand_str(std::size_t strSize); +/** + * @brief Helper function used to replace spaces and zeros by '0' char in the supplied checksum buffer. + * @param checksum Input buffer containing the checksum. + */ +inline void replace_checksum_zeros(char* checksum) { + for (int i=2; i<18; i++) { + if (checksum[i]==' ' || checksum[i]==0) { + checksum[i]='0'; + } + } +} + +/** + * @brief Generates a ProxySQL checksum as a string from the supplied integer hash. + * @param hash The integer hash to be formated as a string. + * @return String representation of the supplied hash. + */ +std::string get_checksum_from_hash(uint64_t hash); + +/** + * @brief Remove the rows from the resultset matching the supplied predicate. + * @param resultset The resultset which rows are to be removed. + * @param pred Predicate that should return 'true' for the rows to be removed. + */ +void remove_sqlite3_resultset_rows( + std::unique_ptr& resultset, const std::function& pred +); + #endif diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 353d7e4510..c996b320bf 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1621,6 +1621,177 @@ void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& m CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); } +const char MYSQL_SERVERS_CHECKSUM_QUERY[] { + "SELECT hostgroup_id, hostname, port, gtid_port, CASE WHEN status=0 OR status=1 OR status=4 THEN 0 ELSE status END status," + " weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers" + " WHERE status<>3 ORDER BY hostgroup_id, hostname, port" +}; + +/** + * @brief Generates a resultset which is used to compute the current 'mysql_servers' checksum. + * @details The resultset should report all servers status as ONLINE(0), with the exception of 'OFFLINE_HARD'. + * Servers with this status should be excluded from the resultset. + * @param mydb The db in which to perform the query, typically 'MySQL_HostGroups_Manager::mydb'. + * @return An SQLite3 resultset for the query 'MYSQL_SERVERS_CHECKSUM_QUERY'. + */ +unique_ptr get_mysql_servers_checksum_resultset(SQLite3DB* mydb) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + mydb->execute_statement(MYSQL_SERVERS_CHECKSUM_QUERY, &error, &cols, &affected_rows, &resultset); + + if (error) { + proxy_error("Checksum generation query for 'mysql_servers' failed with error '%s'\n", error); + assert(0); + } + + return unique_ptr(resultset); +} + +/** + * @brief Generates a resultset holding the current Admin 'runtime_mysql_servers' as reported by Admin. + * @param mydb The db in which to perform the query, typically 'MySQL_HostGroups_Manager::mydb'. + * @return An SQLite3 resultset for the query 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. + */ +unique_ptr get_admin_runtime_mysql_servers(SQLite3DB* mydb) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + return unique_ptr(resultset); +} + +/** + * @brief Removes rows with 'OFFLINE_HARD' servers in the supplied resultset. + * @details It assumes that the supplied resultset is generated via 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. + * @param resultset The resultset from which rows are to be removed. + */ +void remove_resultset_offline_hard_servers(unique_ptr& resultset) { + if (resultset->columns < 5) { + return; + } + + const auto is_offline = [] (SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + return true; + } else { + return false; + } + }; + + remove_sqlite3_resultset_rows(resultset, is_offline); +} + +/** + * @brief Updates the global 'mysql_servers' checksum. + * @details If the new computed checksum matches the supplied 'cluster_checksum', the epoch used for the + * checksum, is the supplied epoch instead of current time. This way we ensure the preservation of the + * checksum and epoch fetched from the ProxySQL cluster peer node. + * + * @param new_checksum The new computed checksum by ProxySQL. + * @param old_checksum A checksum, previously fetched from ProxySQL cluster. Should be left empty if the + * update isn't considering this scenario. + * @param epoch The epoch to be preserved in case the supplied 'cluster_checksum' matches the new computed + * checksum. + */ +void update_glovars_mysql_servers_checksum( + const std::string& new_checksum, const std::string& cluster_checksum = "", const time_t epoch = 0 +) { + GloVars.checksums_values.mysql_servers.set_checksum(const_cast(new_checksum.c_str())); + GloVars.checksums_values.mysql_servers.version++; + + time_t t = time(NULL); + bool computed_checksum_matches { + cluster_checksum != "" && GloVars.checksums_values.mysql_servers.checksum == cluster_checksum + }; + + if (epoch != 0 && computed_checksum_matches) { + GloVars.checksums_values.mysql_servers.epoch = epoch; + } else { + GloVars.checksums_values.mysql_servers.epoch = t; + } + + GloVars.checksums_values.updates_cnt++; + GloVars.generate_global_checksum(); + GloVars.epoch_version = t; +} + +void MySQL_HostGroups_Manager::commit_generate_mysql_servers_table(SQLite3_result* runtime_mysql_servers) { + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + if (runtime_mysql_servers == nullptr) { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + + // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster nodes, or + // relevant for checksum computation. If this step isn't performed, this could cause mismatching + // checksums between different primary nodes in a ProxySQL cluster, since OFFLINE_HARD servers + // preservation depends on unknown and unpredictable connections conditions. + remove_resultset_offline_hard_servers(resultset); + save_runtime_mysql_servers(resultset.release()); + } else { + save_runtime_mysql_servers(runtime_mysql_servers); + } +} + +void MySQL_HostGroups_Manager::commit_update_checksum_from_mysql_servers(SpookyHash& myhash, bool& init) { + unique_ptr mysrvs_checksum_resultset { get_mysql_servers_checksum_resultset(mydb) }; + + // Reset table checksum value before recomputing + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = 0; + + if (mysrvs_checksum_resultset) { + if (mysrvs_checksum_resultset->rows_count) { + if (init == false) { + init = true; + myhash.Init(19,3); + } + uint64_t hash1_ = mysrvs_checksum_resultset->raw_checksum(); + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; + + myhash.Update(&hash1_, sizeof(hash1_)); + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); + } + } else { + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); + } +} + +std::string MySQL_HostGroups_Manager::gen_global_mysql_servers_checksum() { + SpookyHash global_hash; + bool init = false; + + // Regenerate 'mysql_servers' and generate new checksum, initialize the new 'global_hash' + commit_update_checksum_from_mysql_servers(global_hash, init); + + // Complete the hash with the rest of the unchanged modules + for (size_t i = 0; i < table_resultset_checksum.size(); i++) { + uint64_t hash_val = table_resultset_checksum[i]; + + if (i != HGM_TABLES::MYSQL_SERVERS && hash_val != 0) { + if (init == false) { + init = true; + global_hash.Init(19, 3); + } + + global_hash.Update(&hash_val, sizeof(hash_val)); + } + } + + uint64_t hash_1 = 0, hash_2 = 0; + if (init) { + global_hash.Final(&hash_1,&hash_2); + } + + string mysrvs_checksum { get_checksum_from_hash(hash_1) }; + return mysrvs_checksum; +} + bool MySQL_HostGroups_Manager::commit( SQLite3_result* runtime_mysql_servers, const std::string& checksum, const time_t epoch ) { diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 03ad962389..1404cc13df 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -964,17 +964,6 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } } -std::string get_checksum_from_hash(uint64_t hash) { - uint32_t d32[2] = { 0 }; - memcpy(&d32, &hash, sizeof(hash)); - - vector s_buf(20, 0); - sprintf(&s_buf[0],"0x%0X%0X", d32[0], d32[1]); - replace_checksum_zeros(&s_buf[0]); - - return string { &s_buf.front() }; -} - /** * @brief Computes the checksum from a MySQL resultset in the same we already do in 'SQLite3_result::raw_checksum'. * @details For each received column computing the field length via 'strlen' is required, this is because we diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index f30f1e5b8d..933b3600d0 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -14,7 +14,9 @@ #include #include +using std::function; using std::string; +using std::unique_ptr; using std::vector; __attribute__((__format__ (__printf__, 1, 2))) @@ -436,3 +438,22 @@ string rand_str(std::size_t strSize) { return random_str; } } + +std::string get_checksum_from_hash(uint64_t hash) { + uint32_t d32[2] = { 0 }; + memcpy(&d32, &hash, sizeof(hash)); + + vector s_buf(ProxySQL_Checksum_Value_LENGTH, 0); + sprintf(&s_buf[0],"0x%0X%0X", d32[0], d32[1]); + replace_checksum_zeros(&s_buf[0]); + + return string { &s_buf.front() }; +} + +void remove_sqlite3_resultset_rows( + unique_ptr& resultset, const function& pred +) { + const auto remove_it { std::remove_if(resultset->rows.begin(), resultset->rows.end(), pred) }; + resultset->rows.erase(remove_it, resultset->rows.end()); + resultset->rows_count = resultset->rows.size(); +} From 6ec2bbc1d29474f177d55f3b266021ccfa8f4a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 28 Jun 2023 17:04:07 +0200 Subject: [PATCH 34/56] Update 'mysql_servers' checksum when adding auto-discovered GR servers --- lib/MySQL_HostGroups_Manager.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c996b320bf..169cf0305e 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -5889,6 +5889,20 @@ void MySQL_HostGroups_Manager::update_group_replication_add_autodiscovered( proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); generate_mysql_servers_table(); + + // Update the global checksums after 'mysql_servers' regeneration + { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + remove_resultset_offline_hard_servers(resultset); + save_runtime_mysql_servers(resultset.release()); + + string mysrvs_checksum { gen_global_mysql_servers_checksum() }; + proxy_info("New computed global checksum for 'mysql_servers' is '%s'\n", mysrvs_checksum.c_str()); + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + update_table_mysql_servers_for_monitor(false); generate_mysql_group_replication_hostgroups_monitor_resultset(); } From 9e3ab519e83565cf4e54b789941f30e5ff1395ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 28 Jun 2023 18:16:02 +0200 Subject: [PATCH 35/56] Add helper functions for server creation/destruction in hostgroup --- include/MySQL_HostGroups_Manager.h | 49 ++++++++++++++++++ lib/MySQL_HostGroups_Manager.cpp | 80 ++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 239af7be6f..11ec9f37af 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -432,6 +432,27 @@ enum REPLICATION_LAG_SERVER_T { RLS__SIZE }; +/** + * @brief Contains the minimal info for server creation. + */ +struct srv_info_t { + /* @brief Server address */ + string addr; + /* @brief Server port */ + uint16_t port; + /* @brief Server type identifier, used for logging, e.g: 'Aurora AWS', 'GR', etc... */ + string kind; +}; + +/** + * @brief Contains options to be specified during server creation. + */ +struct srv_opts_t { + int64_t weigth; + int64_t max_conns; + int32_t use_ssl; +}; + class MySQL_HostGroups_Manager { private: SQLite3DB *admindb; @@ -801,6 +822,34 @@ class MySQL_HostGroups_Manager { MyHGC * MyHGC_lookup(unsigned int); void MyConn_add_to_pool(MySQL_Connection *); + /** + * @brief Creates a new server in the target hostgroup if isn't already present. + * @details If the server is found already in the target hostgroup, no action is taken, unless its status + * is 'OFFLINE_HARD'. In case of finding it as 'OFFLINE_HARD': + * 1. Server hostgroup attributes are reset to known values, so they can be updated. + * 2. Server attributes are updated to either table definition values, or hostgroup 'servers_defaults'. + * 3. Server is bring back as 'ONLINE'. + * @param hid The hostgroup in which the server is to be created (or to bring it back as 'ONLINE'). + * @param srv_info Basic server info to be used during creation. + * @param srv_opts Server creation options. + * @return 0 in case of success, -1 in case of failure. + */ + int create_new_server_in_hg(uint32_t hid, const srv_info_t& srv_info, const srv_opts_t& srv_opts); + /** + * @brief Completely removes server from the target hostgroup if found. + * @details Several actions are taken if server is found: + * - Set the server as 'OFFLINE_HARD'. + * - Drop all current FREE connections to the server. + * - Delete the server from the 'myhgm.mysql_servers' table. + * + * This later step is not required if the caller is already going to perform a full deletion of the + * servers in the target hostgroup. Which is a common operation during table regeneration. + * @param hid Target hostgroup id. + * @param addr Target server address. + * @param port Target server port. + * @return 0 in case of success, -1 in case of failure. + */ + int remove_server_in_hg(uint32_t hid, const string& addr, uint16_t port); MySQL_Connection * get_MyConn_from_pool(unsigned int hid, MySQL_Session *sess, bool ff, char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index d6de5d1fb6..dd3f17d975 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7558,6 +7558,86 @@ bool MySQL_HostGroups_Manager::aws_aurora_replication_lag_action(int _whid, int return ret; } +int MySQL_HostGroups_Manager::create_new_server_in_hg( + uint32_t hid, const srv_info_t& srv_info, const srv_opts_t& srv_opts +) { + int32_t res = -1; + MySrvC* mysrvc = find_server_in_hg(hid, srv_info.addr, srv_info.port); + + if (mysrvc == nullptr) { + char* c_hostname { const_cast(srv_info.addr.c_str()) }; + MySrvC* mysrvc = new MySrvC( + c_hostname, srv_info.port, 0, srv_opts.weigth, MYSQL_SERVER_STATUS_ONLINE, 0, srv_opts.max_conns, 0, + srv_opts.use_ssl, 0, const_cast("") + ); + add(mysrvc,hid); + proxy_info( + "Adding new discovered %s node %s:%d with: hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + srv_info.kind.c_str(), c_hostname, srv_info.port, hid, mysrvc->weight, mysrvc->max_connections, + mysrvc->use_ssl + ); + + res = 0; + } else { + // If the server is found as 'OFFLINE_HARD' we reset the 'MySrvC' values corresponding with the + // 'servers_defaults' (as in a new 'MySrvC' creation). We then later update these values with the + // 'servers_defaults' attributes from its corresponding 'MyHGC'. This way we ensure uniform behavior + // of new servers, and 'OFFLINE_HARD' ones when a user update 'servers_defaults' values, and reloads + // the servers to runtime. + if (mysrvc && mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + reset_hg_attrs_server_defaults(mysrvc); + update_hg_attrs_server_defaults(mysrvc, mysrvc->myhgc); + mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; + + proxy_info( + "Found healthy previously discovered %s node %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:" + " hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + srv_info.kind.c_str(), srv_info.addr.c_str(), srv_info.port, hid, mysrvc->weight, + mysrvc->max_connections, mysrvc->use_ssl + ); + + res = 0; + } + } + + return res; +} + +int MySQL_HostGroups_Manager::remove_server_in_hg(uint32_t hid, const string& addr, uint16_t port) { + MySrvC* mysrvc = find_server_in_hg(hid, addr, port); + if (mysrvc == nullptr) { + return -1; + } + + uint64_t mysrvc_addr = reinterpret_cast(mysrvc); + + proxy_warning( + "Removed server at address %ld, hostgroup %d, address %s port %d." + " Setting status OFFLINE HARD and immediately dropping all free connections." + " Used connections will be dropped when trying to use them\n", + mysrvc_addr, hid, mysrvc->address, mysrvc->port + ); + + // Set the server status + mysrvc->status=MYSQL_SERVER_STATUS_OFFLINE_HARD; + mysrvc->ConnectionsFree->drop_all_connections(); + + // TODO-NOTE: This is only required in case the caller isn't going to perform: + // - Full deletion of servers in the target 'hid'. + // - Table regeneration for the servers in the target 'hid'. + // This is a very common pattern when further operations have been performed over the + // servers, e.g. a set of servers additions and deletions over the target hostgroups. + // //////////////////////////////////////////////////////////////////////// + + // Remove the server from the table + const string del_srv_query { "DELETE FROM mysql_servers WHERE mem_pointer=" + std::to_string(mysrvc_addr) }; + mydb->execute(del_srv_query.c_str()); + + // //////////////////////////////////////////////////////////////////////// + + return 0; +} + // FIXME: complete this!! void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid, char *_server_id, bool verbose) { int cols=0; From 4524bcba6ce39847082801ad018e763c5840e9eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 29 Jun 2023 09:10:06 +0200 Subject: [PATCH 36/56] Fix AWS Aurora replicas not being SHUNNED due to replication lag Since the same 'AWS_Aurora_status_entry' index was being used for computing the previous and current statuses, no difference was being found, and no action taken. --- lib/MySQL_Monitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 234b1ab5fa..0087f94e27 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6493,7 +6493,7 @@ void MySQL_Monitor::evaluate_aws_aurora_results(unsigned int wHG, unsigned int r if (strcmp(prev_hse->server_id,hse->server_id)==0) { bool prev_enabled = true; - unsigned int prev_lag_ms = estimate_lag(hse->server_id, lasts_ase, ase_idx, add_lag_ms, min_lag_ms, lag_num_checks); + unsigned int prev_lag_ms = estimate_lag(hse->server_id, lasts_ase, prev_ase_idx, add_lag_ms, min_lag_ms, lag_num_checks); if (prev_lag_ms > max_latency_ms) { prev_enabled = false; } From 72cfd2452aec689880d2de03fde0eee1aea3d70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 29 Jun 2023 09:15:21 +0200 Subject: [PATCH 37/56] Fix AWS Aurora new writer not honoring 'new_reader_weight' When a writer is also placed in the reader hostgroup due to 'writer_is_also_reader', this new reader should also honor 'new_reader_weight'. --- lib/MySQL_HostGroups_Manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index dd3f17d975..af7173fa38 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7799,6 +7799,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid mydb->execute(query); q = (char *)"UPDATE mysql_servers_incoming SET weight=%d WHERE hostgroup_id=%d AND hostname='%s%s' AND port=%d"; sprintf(query, q, new_reader_weight, read_HG, _server_id, domain_name, aurora_port); + mydb->execute(query); } uint64_t checksum_current = 0; uint64_t checksum_incoming = 0; From 2d5359a4a044c1e56de5da8a4853c2468df7a113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 29 Jun 2023 09:41:36 +0200 Subject: [PATCH 38/56] Fix invalid servers removal when present in multiple AWS Aurora clusters --- lib/MySQL_HostGroups_Manager.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index af7173fa38..5931c268b5 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7957,9 +7957,9 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid } pthread_mutex_unlock(&AWS_Aurora_Info_mutex); } - q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE hostname='%s%s' AND port=%d AND status<>3"; - query=(char *)malloc(strlen(q)+strlen(_server_id)+strlen(domain_name)+32); - sprintf(query, q, _server_id, domain_name, aurora_port); + q = (char*)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE hostname='%s%s' AND port=%d AND status<>3 AND hostgroup_id IN (%d,%d)"; + query=(char *)malloc(strlen(q)+strlen(_server_id)+strlen(domain_name)+32+32+32); + sprintf(query, q, _server_id, domain_name, aurora_port, _whid, _rhid); mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset); if (error) { free(error); @@ -7972,14 +7972,17 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid GloAdmin->mysql_servers_wrlock(); mydb->execute("DELETE FROM mysql_servers_incoming"); mydb->execute("INSERT INTO mysql_servers_incoming SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers"); - q=(char *)"UPDATE OR IGNORE mysql_servers_incoming SET hostgroup_id=%d WHERE hostname='%s%s' AND port=%d AND hostgroup_id<>%d"; + // If server present as WRITER try moving it to 'reader_hostgroup'. + q=(char *)"UPDATE OR IGNORE mysql_servers_incoming SET hostgroup_id=%d WHERE hostname='%s%s' AND port=%d AND hostgroup_id=%d"; query=(char *)malloc(strlen(q)+strlen(_server_id)+strlen(domain_name)+512); - sprintf(query, q, _rhid, _server_id, domain_name, aurora_port, _rhid); + sprintf(query, q, _rhid, _server_id, domain_name, aurora_port, _whid); mydb->execute(query); //free(query); - q=(char *)"DELETE FROM mysql_servers_incoming WHERE hostname='%s%s' AND port=%d AND hostgroup_id<>%d"; + // Reader could previously be also a reader, in which case previous operation 'UPDATE OR IGNORE' + // did nothing. If server is still in the 'writer_hostgroup', we should remove it. + q=(char *)"DELETE FROM mysql_servers_incoming WHERE hostname='%s%s' AND port=%d AND hostgroup_id=%d"; //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); - sprintf(query, q, _server_id, domain_name, aurora_port, _rhid); + sprintf(query, q, _server_id, domain_name, aurora_port, _whid); mydb->execute(query); //free(query); q=(char *)"UPDATE mysql_servers_incoming SET status=0 WHERE hostname='%s%s' AND port=%d AND hostgroup_id=%d"; From 813355c3d45c36c40480b046148f9fdf1ef08320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 29 Jun 2023 16:09:21 +0200 Subject: [PATCH 39/56] Honor hostgroup attributes for AWS Aurora auto-discovery - Refactor of 'commit' checksums computation. - Refactor update of 'AWS_Aurora_Hosts_resultset'. - Fix invalid use of SHUNNED|SHUNNED_REPLICATION_LAG servers for checksum computation via 'MYSQL_SERVERS_CHECKSUM_QUERY'. --- include/MySQL_HostGroups_Manager.h | 13 ++ lib/MySQL_HostGroups_Manager.cpp | 313 ++++++++++++----------------- 2 files changed, 141 insertions(+), 185 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 11ec9f37af..ff62f3501d 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -940,6 +940,19 @@ class MySQL_HostGroups_Manager { bool aws_aurora_replication_lag_action(int _whid, int _rhid, char *server_id, float current_replication_lag_ms, bool enable, bool is_writer, bool verbose=true); void update_aws_aurora_set_writer(int _whid, int _rhid, char *server_id, bool verbose=true); void update_aws_aurora_set_reader(int _whid, int _rhid, char *server_id); + /** + * @brief Updates the resultset and corresponding checksum used by Monitor for AWS Aurora. + * @details This is required to be called when: + * - The 'mysql_aws_aurora_hostgroups' table is regenerated (via 'commit'). + * - When new servers are discovered, and created in already monitored Aurora clusters. + * + * The resultset holds the servers that are present in 'mysql_servers' table, and share hostgroups with + * the **active** clusters specified in 'mysql_aws_aurora_hostgroups'. See query + * 'SELECT_AWS_AURORA_SERVERS_FOR_MONITOR'. + * @param lock Wether if both 'AWS_Aurora_Info_mutex' and 'MySQL_Monitor::aws_aurora_mutex' mutexes should + * be taken or not. + */ + void update_aws_aurora_hosts_monitor_resultset(bool lock=false); SQLite3_result * get_stats_mysql_gtid_executed(); void generate_mysql_gtid_executed_tables(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 5931c268b5..618b333f9e 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1614,6 +1614,11 @@ void MySQL_HostGroups_Manager::CUCFT1(SpookyHash& myhash, bool& init, const stri } void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& myhash, bool& init) { + // Always reset the current table values before recomputing + for (size_t i = 0; i < table_resultset_checksum.size(); i++) { + if (i != HGM_TABLES::MYSQL_SERVERS) { table_resultset_checksum[i] = 0; } + } + CUCFT1(myhash,init,"mysql_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); CUCFT1(myhash,init,"mysql_group_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GROUP_REPLICATION_HOSTGROUPS]); CUCFT1(myhash,init,"mysql_galera_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GALERA_HOSTGROUPS]); @@ -2044,94 +2049,24 @@ bool MySQL_HostGroups_Manager::commit( // Checksums are always generated - 'admin-checksum_*' deprecated { - uint64_t hash1 = 0, hash2 = 0; + commit_generate_mysql_servers_table(runtime_mysql_servers); + SpookyHash myhash; - char buf[ProxySQL_Checksum_Value_LENGTH]; bool init = false; - { - mydb->execute("DELETE FROM mysql_servers"); - generate_mysql_servers_table(); - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (runtime_mysql_servers == nullptr) { - char* error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - - mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); - - // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster - // nodes, or relevant for checksum computation. - const size_t init_row_count = resultset->rows_count; - size_t rm_rows_count = 0; - const auto is_offline_server = [&rm_rows_count] (SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - rm_rows_count += 1; - return true; - } else { - return false; - } - }; - resultset->rows.erase( - std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), - resultset->rows.end() - ); - resultset->rows_count = init_row_count - rm_rows_count; - - save_runtime_mysql_servers(resultset); - } else { - save_runtime_mysql_servers(runtime_mysql_servers); - } - - // reset all checksum - table_resultset_checksum.fill(0); - - if (resultset) { - if (resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = resultset->raw_checksum(); - - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; - - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); - } - delete resultset; - } else { - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); - } - } + commit_update_checksum_from_mysql_servers(myhash, init); commit_update_checksums_from_tables(myhash, init); + uint64_t hash1 = 0, hash2 = 0; if (init == true) { myhash.Final(&hash1, &hash2); } - uint32_t d32[2]; - memcpy(&d32,&hash1,sizeof(hash1)); - sprintf(buf,"0x%0X%0X", d32[0], d32[1]); + + string new_checksum { get_checksum_from_hash(hash1) }; + proxy_info("New computed global checksum for 'mysql_servers' is '%s'\n", new_checksum.c_str()); + pthread_mutex_lock(&GloVars.checksum_mutex); - GloVars.checksums_values.mysql_servers.set_checksum(buf); - GloVars.checksums_values.mysql_servers.version++; - //struct timespec ts; - //clock_gettime(CLOCK_REALTIME, &ts); - time_t t = time(NULL); - if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_servers.checksum == checksum) { - GloVars.checksums_values.mysql_servers.epoch = epoch; - } else { - GloVars.checksums_values.mysql_servers.epoch = t; - } - GloVars.checksums_values.updates_cnt++; - GloVars.generate_global_checksum(); - GloVars.epoch_version = t; + update_glovars_mysql_servers_checksum(new_checksum, checksum, epoch); pthread_mutex_unlock(&GloVars.checksum_mutex); } @@ -7429,23 +7364,7 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { // it is now time to build a new structure in Monitor pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl , max_lag_ms , check_interval_ms , check_timeout_ms , " - "add_lag_ms , min_lag_ms , lag_num_checks FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR " - "hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3) GROUP BY writer_hostgroup, hostname, port"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (resultset) { - if (GloMyMon->AWS_Aurora_Hosts_resultset) { - delete GloMyMon->AWS_Aurora_Hosts_resultset; - } - GloMyMon->AWS_Aurora_Hosts_resultset=resultset; - GloMyMon->AWS_Aurora_Hosts_resultset_checksum=resultset->raw_checksum(); - } - } + update_aws_aurora_hosts_monitor_resultset(false); pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); pthread_mutex_unlock(&AWS_Aurora_Info_mutex); @@ -7879,45 +7798,57 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid free(query); query = NULL; } else { + string full_hostname { string { _server_id } + string { domain_name } }; + GloAdmin->mysql_servers_wrlock(); - mydb->execute("DELETE FROM mysql_servers_incoming"); - q=(char *)"INSERT INTO mysql_servers_incoming SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE hostname<>'%s%s'"; - sprintf(query,q, _server_id, domain_name); - mydb->execute(query); + wrlock(); - unsigned int max_max_connections = 1000; - unsigned int max_use_ssl = 0; - MyHGC *myhgc = MyHGC_lookup(_whid); - for (int j = 0; j < (int) myhgc->mysrvs->cnt(); j++) { - MySrvC *mysrvc = (MySrvC *) myhgc->mysrvs->servers->index(j); - if (mysrvc->max_connections > max_max_connections) { - max_max_connections = mysrvc->max_connections; - } - if (mysrvc->use_ssl > max_use_ssl) { - max_use_ssl = mysrvc->use_ssl; - } + srv_info_t srv_info { full_hostname, static_cast(aurora_port), "Aurora AWS" }; + srv_opts_t wr_srv_opts { -1, -1, -1 }; + + int wr_res = create_new_server_in_hg(_writer_hostgroup, srv_info, wr_srv_opts); + int rd_res = -1; + + // WRITER can also be placed as READER, or could previously be one + if (writer_is_also_reader && read_HG >= 0) { + srv_opts_t rd_srv_opts { new_reader_weight, -1, -1 }; + rd_res = create_new_server_in_hg(read_HG, srv_info, rd_srv_opts); } - q=(char *)"INSERT INTO mysql_servers_incoming (hostgroup_id, hostname, port, weight, max_connections, use_ssl) VALUES (%d, '%s%s', %d, %d, %d, %d)"; - sprintf(query,q, _writer_hostgroup, _server_id, domain_name, aurora_port, new_reader_weight, max_max_connections, max_use_ssl); - mydb->execute(query); - if (writer_is_also_reader && read_HG>=0) { - q=(char *)"INSERT INTO mysql_servers_incoming (hostgroup_id, hostname, port, weight, max_connections, use_ssl) VALUES (%d, '%s%s', %d, %d, %d, %d)"; - sprintf(query, q, read_HG, _server_id, domain_name, aurora_port, new_reader_weight, max_max_connections, max_use_ssl); - mydb->execute(query); + // A new server has been created, or an OFFLINE_HARD brought back as ONLINE + if (wr_res == 0 || rd_res == 0) { + proxy_info( + "AWS Aurora: setting new auto-discovered host %s:%d as writer\n", full_hostname.c_str(), aurora_port + ); + purge_mysql_servers_table(); + + const char del_srvs_query_t[] { "DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)" }; + const string del_srvs_query { cstr_format(del_srvs_query_t, _whid, _rhid).str }; + mydb->execute(del_srvs_query.c_str()); + + generate_mysql_servers_table(&_whid); + generate_mysql_servers_table(&_rhid); + + // Update the global checksums after 'mysql_servers' regeneration + { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + remove_resultset_offline_hard_servers(resultset); + save_runtime_mysql_servers(resultset.release()); + + string mysrvs_checksum { gen_global_mysql_servers_checksum() }; + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + + // Because 'commit' isn't called, we are required to update 'mysql_servers_for_monitor'. + update_table_mysql_servers_for_monitor(false); + // Update AWS Aurora resultset used for monitoring + update_aws_aurora_hosts_monitor_resultset(true); } - proxy_info("AWS Aurora: setting new auto-discovered host %s%s:%d as writer\n", _server_id, domain_name, aurora_port); - commit(); - wrlock(); - q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)"; - sprintf(query,q,_whid,_rhid); - mydb->execute(query); - generate_mysql_servers_table(&_whid); - generate_mysql_servers_table(&_rhid); + wrunlock(); GloAdmin->mysql_servers_wrunlock(); - free(query); - query = NULL; } } if (resultset) { @@ -8029,70 +7960,46 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid } else { // we couldn't find the server // autodiscovery algorithm here - char *full_hostname=(char *)malloc(strlen(_server_id)+strlen(domain_name)+1); - sprintf(full_hostname, "%s%s", _server_id, domain_name); - bool found = false; + string full_hostname { string { _server_id } + string { domain_name } }; GloAdmin->mysql_servers_wrlock(); - unsigned int max_max_connections = 10; - unsigned int max_use_ssl = 0; wrlock(); - MyHGC *myhgc=MyHGC_lookup(_rhid); - { - for (int j=0; j<(int)myhgc->mysrvs->cnt(); j++) { - MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j); - if (mysrvc->max_connections > max_max_connections) { - max_max_connections = mysrvc->max_connections; - } - if (mysrvc->use_ssl > max_use_ssl) { - max_use_ssl = mysrvc->use_ssl; - } - if (strcmp(mysrvc->address,full_hostname)==0 && mysrvc->port==aurora_port) { - found = true; - // we found the server, we just configure it online if it was offline - if (mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) { - mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; - } - } - } - if (found == false) { // the server doesn't exist - MySrvC *mysrvc=new MySrvC(full_hostname, aurora_port, 0, new_reader_weight, MYSQL_SERVER_STATUS_ONLINE, 0, max_max_connections, 0, max_use_ssl, 0, (char *)""); // add new fields here if adding more columns in mysql_servers - proxy_info("Adding new discovered AWS Aurora node %s:%d with: hostgroup=%d, weight=%d, max_connections=%d\n" , full_hostname, aurora_port, _rhid , new_reader_weight, max_max_connections); - add(mysrvc,_rhid); - } - q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)"; - query = (char *)malloc(strlen(q)+64); - sprintf(query,q,_whid,_rhid); - mydb->execute(query); + + srv_info_t srv_info { full_hostname, static_cast(aurora_port), "Aurora AWS" }; + srv_opts_t srv_opts { new_reader_weight, -1, -1 }; + int wr_res = create_new_server_in_hg(_rhid, srv_info, srv_opts); + + // A new server has been created, or an OFFLINE_HARD brought back as ONLINE + if (wr_res == 0) { + purge_mysql_servers_table(); + + const char del_srvs_query_t[] { "DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)" }; + const string del_srvs_query { cstr_format(del_srvs_query_t, _whid, _rhid).str }; + mydb->execute(del_srvs_query.c_str()); + generate_mysql_servers_table(&_whid); generate_mysql_servers_table(&_rhid); - free(query); - } - // NOTE: Because 'commit' isn't called, we are required to update 'mysql_servers_for_monitor'. - // Also note that 'generate_mysql_servers' is previously called. - update_table_mysql_servers_for_monitor(false); - wrunlock(); - // it is now time to build a new structure in Monitor - pthread_mutex_lock(&AWS_Aurora_Info_mutex); - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl , max_lag_ms , check_interval_ms , check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3) GROUP BY hostname, port"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (resultset) { - if (GloMyMon->AWS_Aurora_Hosts_resultset) { - delete GloMyMon->AWS_Aurora_Hosts_resultset; - } - GloMyMon->AWS_Aurora_Hosts_resultset=resultset; - GloMyMon->AWS_Aurora_Hosts_resultset_checksum=resultset->raw_checksum(); + + // Update the global checksums after 'mysql_servers' regeneration + { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + remove_resultset_offline_hard_servers(resultset); + save_runtime_mysql_servers(resultset.release()); + + string mysrvs_checksum { gen_global_mysql_servers_checksum() }; + proxy_info("New computed global checksum for 'mysql_servers' is '%s'\n", mysrvs_checksum.c_str()); + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); } + + // Because 'commit' isn't called, we are required to update 'mysql_servers_for_monitor'. + update_table_mysql_servers_for_monitor(false); + // Update AWS Aurora resultset used for monitoring + update_aws_aurora_hosts_monitor_resultset(true); } - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - pthread_mutex_unlock(&AWS_Aurora_Info_mutex); + + wrunlock(); GloAdmin->mysql_servers_wrunlock(); - free(full_hostname); } } if (resultset) { @@ -8102,6 +8009,42 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid free(domain_name); } +const char SELECT_AWS_AURORA_SERVERS_FOR_MONITOR[] { + "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, max_lag_ms, check_interval_ms," + " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks FROM mysql_servers" + " JOIN mysql_aws_aurora_hostgroups ON" + " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" + " GROUP BY writer_hostgroup, hostname, port" +}; + +void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lock) { + if (lock) { + pthread_mutex_lock(&AWS_Aurora_Info_mutex); + pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); + } + + SQLite3_result* resultset = nullptr; + { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + mydb->execute_statement(SELECT_AWS_AURORA_SERVERS_FOR_MONITOR, &error, &cols, &affected_rows, &resultset); + } + + if (resultset) { + if (GloMyMon->AWS_Aurora_Hosts_resultset) { + delete GloMyMon->AWS_Aurora_Hosts_resultset; + } + GloMyMon->AWS_Aurora_Hosts_resultset=resultset; + GloMyMon->AWS_Aurora_Hosts_resultset_checksum=resultset->raw_checksum(); + } + + if (lock) { + pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); + pthread_mutex_unlock(&AWS_Aurora_Info_mutex); + } +} + MySrvC* MySQL_HostGroups_Manager::find_server_in_hg(unsigned int _hid, const std::string& addr, int port) { MySrvC* f_server = nullptr; From 3d06427efe44698eac50dfb8d54e699a325bb1b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 30 Jun 2023 13:02:25 +0200 Subject: [PATCH 40/56] Improve simulator support for AWS Aurora - Preserve old simulator behavior via 'TEST_AURORA_RANDOM' macro. - Add new deterministic behavior for testing particular scenarios. - Add support for simulating multiple clusters via an updated 'REPLICA_HOST_STATUS' table and hostgroup based selection. - Provide a sensible default population for manual testing. --- Makefile | 15 +- include/MySQL_HostGroups_Manager.h | 18 ++ include/MySQL_Monitor.hpp | 8 + include/SQLite3_Server.h | 7 +- include/proxysql_admin.h | 2 + include/sqlite3db.h | 2 + lib/MySQL_Monitor.cpp | 124 +++--------- lib/ProxySQL_Admin.cpp | 38 ++-- src/SQLite3_Server.cpp | 290 +++++++++++++++++++++++++---- 9 files changed, 349 insertions(+), 155 deletions(-) diff --git a/Makefile b/Makefile index 8308717948..c9f8e11cb7 100644 --- a/Makefile +++ b/Makefile @@ -76,8 +76,11 @@ debug: build_deps_debug build_lib_debug build_src_debug .PHONY: testaurora testaurora: build_deps_debug build_lib_testaurora build_src_testaurora - cd test/tap && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE} - cd test/tap/tests && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS) + # cd test/tap && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE} + # cd test/tap/tests && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS) + +.PHONY: testaurora_random +testaurora_random: build_deps_debug build_lib_testaurora_random build_src_testaurora_random .PHONY: testgalera testgalera: build_deps_debug build_lib_testgalera build_src_testgalera @@ -128,10 +131,18 @@ build_lib_debug: build_deps_debug build_src_testaurora: build_deps build_lib_testaurora cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE} +.PHONY: build_src_testaurora_random +build_src_testaurora_random: build_deps build_lib_testaurora_random + cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_AURORA_RANDOM" CC=${CC} CXX=${CXX} ${MAKE} + .PHONY: build_lib_testaurora build_lib_testaurora: build_deps_debug cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE} +.PHONY: build_lib_testaurora_random +build_lib_testaurora_random: build_deps_debug + cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_AURORA_RANDOM" CC=${CC} CXX=${CXX} ${MAKE} + .PHONY: build_src_testgalera build_src_testgalera: build_deps build_lib_testgalera cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GALERA" CC=${CC} CXX=${CXX} ${MAKE} diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index ff62f3501d..bb8adaeedf 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -555,8 +555,26 @@ class MySQL_HostGroups_Manager { MySQL_HostGroups_Manager* myHGM; }; + /** + * @brief Used by 'MySQL_Monitor::read_only' to hold a mapping between servers and hostgroups. + * @details The hostgroup mapping holds the MySrvC for each of the hostgroups in which the servers is + * present, distinguishing between 'READER' and 'WRITER' hostgroups. + */ std::unordered_map> hostgroup_server_mapping; + /** + * @brief Holds the previous computed checksum for 'mysql_servers'. + * @details Used to check if the servers checksums has changed during 'commit', if a change is detected, + * the member 'hostgroup_server_mapping' is required to be regenerated. + * + * This is only updated during 'read_only_action_v2', since the action itself modifies + * 'hostgroup_server_mapping' in case any actions needs to be performed against the servers. + */ uint64_t hgsm_mysql_servers_checksum = 0; + /** + * @brief Holds the previous checksum for the 'MYSQL_REPLICATION_HOSTGROUPS'. + * @details Used during 'commit' to determine if config has changed for 'MYSQL_REPLICATION_HOSTGROUPS', + * and 'hostgroup_server_mapping' should be rebuild. + */ uint64_t hgsm_mysql_replication_hostgroups_checksum = 0; diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index d4e5178e80..8454cf939e 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -70,6 +70,14 @@ A single AWS_Aurora_monitor_node will have a AWS_Aurora_status_entry per check. */ +#ifdef TEST_AURORA + +#define TEST_AURORA_MONITOR_BASE_QUERY \ + "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, REPLICA_LAG_IN_MILLISECONDS, CPU"\ + " FROM REPLICA_HOST_STATUS ORDER BY SERVER_ID " + +#endif + class AWS_Aurora_replica_host_status_entry { public: char * server_id = NULL; diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 818c6971fc..fe304c9a94 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -71,7 +71,12 @@ class SQLite3_Server { unsigned int num_aurora_servers[3]; unsigned int max_num_aurora_servers; pthread_mutex_t aurora_mutex; - void populate_aws_aurora_table(MySQL_Session *sess); + /** + * @brief Handles queries to table 'REPLICA_HOST_STATUS'. + * @details This function needs to be called with lock on mutex aurora_mutex already acquired. + * @param sess The session which request is to be handled. + */ + void populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg); void init_aurora_ifaces_string(std::string& s); #endif // TEST_AURORA #ifdef TEST_GALERA diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 24a365294e..cf1f6981d1 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -544,6 +544,8 @@ class ProxySQL_Admin { #ifdef TEST_AURORA void enable_aurora_testing(); + void enable_aurora_testing_populate_mysql_servers(); + void enable_aurora_testing_populate_mysql_aurora_hostgroups(); #endif // TEST_AURORA #ifdef TEST_GALERA diff --git a/include/sqlite3db.h b/include/sqlite3db.h index b4d3c52f3b..977a8cc296 100644 --- a/include/sqlite3db.h +++ b/include/sqlite3db.h @@ -1,9 +1,11 @@ #ifndef __CLASS_SQLITE3DB_H #define __CLASS_SQLITE3DB_H +#include #include "sqlite3.h" #undef swap #undef min #undef max +#include #include #define PROXYSQL_SQLITE3DB_PTHREAD_MUTEX diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 0087f94e27..36372e064b 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -5890,12 +5890,12 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { rnd %= num_hosts; rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); //proxy_info("Looping Monitor thread for AWS Aurora writer HG %u\n", wHG); -#ifdef TEST_AURORA +#ifdef TEST_AURORA_RANDOM if (rand() % 100 < 30) { // we randomly fail 30% of the requests rc_ping = false; } -#endif // TEST_AURORA +#endif // TEST_AURORA_RANDOM if (rc_ping) { found_pingable_host = true; cur_host_idx = rnd; @@ -5918,12 +5918,12 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } -#ifdef TEST_AURORA +#ifdef TEST_AURORA_RANDOM if (rand() % 200 == 0) { // we randomly fail 0.5% of the requests found_pingable_host = false; } -#endif // TEST_AURORA +#endif // TEST_AURORA_RANDOM if (found_pingable_host == false) { proxy_error("No node is pingable for AWS Aurora cluster with writer HG %u\n", wHG); @@ -5977,7 +5977,10 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value #ifdef TEST_AURORA - mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, REPLICA_LAG_IN_MILLISECONDS, CPU FROM REPLICA_HOST_STATUS ORDER BY SERVER_ID"); + { + string query { TEST_AURORA_MONITOR_BASE_QUERY + std::to_string(wHG) }; + mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query.c_str()); + } #else // for reference we list the old queries. // original implementation: @@ -6185,34 +6188,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } -/* - mmsd->writer_hostgroup=atoi(r->fields[0]); - mmsd->writer_is_also_reader=atoi(r->fields[4]); - mmsd->max_transactions_behind=atoi(r->fields[5]); - mmsd->mondb=monitordb; - WorkItem* item; - item=new WorkItem(mmsd,monitor_AWS_Aurora_thread); - GloMyMon->queue.add(item); - usleep(us); -*/ -// } - -/* - for - for (std::vector::iterator it = Galera_Hosts_resultset->rows.begin() ; it != Galera_Hosts_resultset->rows.end(); ++it) { - - } - SQLite3_row *r=*it; - bool rc_ping = true; - rc_ping = server_responds_to_ping(r->fields[1],atoi(r->fields[2])); - if (rc_ping) { // only if server is responding to pings - MySQL_Monitor_State_Data *mmsd=new MySQL_Monitor_State_Data(r->fields[1],atoi(r->fields[2]), NULL, atoi(r->fields[3])); - mmsd->writer_hostgroup=atoi(r->fields[0]); - mmsd->writer_is_also_reader=atoi(r->fields[4]); - mmsd->max_transactions_behind=atoi(r->fields[5]); - mmsd->mondb=monitordb; - -*/ } __exit_monitor_AWS_Aurora_thread_HG_now: if (mmsd) { @@ -6332,66 +6307,6 @@ void * MySQL_Monitor::monitor_aws_aurora() { pthread_mutex_unlock(&aws_aurora_mutex); } -/* - if (t1 < next_loop_at) { - goto __sleep_monitor_aws_aurora; - } - - if (next_loop_at == 0) { - // free the queue - - } - - next_loop_at=t1+1000*mysql_thread___monitor_galera_healthcheck_interval; - pthread_mutex_lock(&aws_aurora_mutex); - if (AWS_Aurora_Hosts_resultset==NULL) { - goto __end_monitor_aws_aurora_loop; - } else { - if (AWS_Aurora_Hosts_resultset->rows_count==0) { - goto __end_monitor_aws_aurora_loop; - } - int us=100; - if (AWS_Aurora_Hosts_resultset->rows_count) { - us=mysql_thread___monitor_read_only_interval/2/Galera_Hosts_resultset->rows_count; - } - for (std::vector::iterator it = Galera_Hosts_resultset->rows.begin() ; it != Galera_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - bool rc_ping = true; - rc_ping = server_responds_to_ping(r->fields[1],atoi(r->fields[2])); - if (rc_ping) { // only if server is responding to pings - MySQL_Monitor_State_Data *mmsd=new MySQL_Monitor_State_Data(r->fields[1],atoi(r->fields[2]), NULL, atoi(r->fields[3])); - mmsd->writer_hostgroup=atoi(r->fields[0]); - mmsd->writer_is_also_reader=atoi(r->fields[4]); - mmsd->max_transactions_behind=atoi(r->fields[5]); - mmsd->mondb=monitordb; - WorkItem* item; - item=new WorkItem(mmsd,monitor_AWS_Aurora_thread); - GloMyMon->queue.add(item); - usleep(us); - } - if (GloMyMon->shutdown) { - pthread_mutex_unlock(&galera_mutex); - return NULL; - } - } - } - -__end_monitor_aws_aurora_loop: - pthread_mutex_unlock(&aws_aurora_mutex); - if (mysql_thread___monitor_enabled==true) { - } - -__sleep_monitor_aws_aurora: - t2=monotonic_time(); - if (t2 200000) { - st = 200000; - } - usleep(st); - } -*/ usleep(10000); } if (mysql_thr) { @@ -6433,6 +6348,18 @@ unsigned int MySQL_Monitor::estimate_lag(char* server_id, AWS_Aurora_status_entr return mlag; } +void print_aws_aurora_status_entry(AWS_Aurora_status_entry* aase) { + if (aase && aase->start_time) { + if (aase->host_statuses->size()) { + for (AWS_Aurora_replica_host_status_entry* hse : *aase->host_statuses) { + if (hse) { + fprintf(stderr,"%s %s %s %f %f\n", hse->server_id, hse->session_id, hse->last_update_timestamp, hse->replica_lag_ms , hse->cpu); + } + } + } + } +} + void MySQL_Monitor::evaluate_aws_aurora_results(unsigned int wHG, unsigned int rHG, AWS_Aurora_status_entry **lasts_ase, unsigned int ase_idx, unsigned int max_latency_ms, unsigned int add_lag_ms, unsigned int min_lag_ms, unsigned int lag_num_checks) { #ifdef TEST_AURORA unsigned int i = 0; @@ -6450,16 +6377,7 @@ void MySQL_Monitor::evaluate_aws_aurora_results(unsigned int wHG, unsigned int r for (i=0; i < N_L_ASE; i++) { AWS_Aurora_status_entry *aase = lasts_ase[i]; if (ev == true || i == ase_idx) { - if (aase && aase->start_time) { - if ( aase->host_statuses->size() ) { - for (std::vector::iterator it3 = aase->host_statuses->begin(); it3!=aase->host_statuses->end(); ++it3) { - AWS_Aurora_replica_host_status_entry *hse = *it3; - if (hse) { - fprintf(stderr,"%s %s %s %f %f\n", hse->server_id, hse->session_id, hse->last_update_timestamp, hse->replica_lag_ms , hse->cpu); - } - } - } - } + print_aws_aurora_status_entry(aase); } } } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index e6f77a8942..de1cc5a67b 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -13923,17 +13923,13 @@ void ProxySQL_Admin::enable_galera_testing() { } #endif // TEST_GALERA #ifdef TEST_AURORA -void ProxySQL_Admin::enable_aurora_testing() { - proxy_info("Admin is enabling AWS Aurora Testing using SQLite3 Server and HGs from 1271 to 1276\n"); + +void ProxySQL_Admin::enable_aurora_testing_populate_mysql_servers() { sqlite3_stmt *statement=NULL; - //sqlite3 *mydb3=admindb->get_db(); unsigned int num_aurora_servers = GloSQLite3Server->num_aurora_servers[0]; - int rc; - mysql_servers_wrlock(); admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 1271 AND 1276"); char *query=(char *)"INSERT INTO mysql_servers (hostgroup_id,hostname,use_ssl,comment) VALUES (?1, ?2, ?3, ?4)"; - //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query, -1, &statement, 0); - rc = admindb->prepare_v2(query, &statement); + int rc = admindb->prepare_v2(query, &statement); ASSERT_SQLITE_OK(rc, admindb); for (unsigned int j=1; j<4; j++) { proxy_info("Admin is enabling AWS Aurora Testing using SQLite3 Server and HGs 127%d and 127%d\n" , j*2-1 , j*2); @@ -13947,7 +13943,6 @@ void ProxySQL_Admin::enable_aurora_testing() { } else { if (j==3) { serverid = "host.1." + std::to_string(i+11) + ".aws-test.com"; - //serverid = "host." + std::to_string(j) + "." + std::to_string(i+11) + ".aws.3.test.com"; } } } @@ -13963,24 +13958,35 @@ void ProxySQL_Admin::enable_aurora_testing() { } } (*proxy_sqlite3_finalize)(statement); +} + +void ProxySQL_Admin::enable_aurora_testing_populate_mysql_aurora_hostgroups() { +#ifndef TEST_AURORA_RANDOM + admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) VALUES (1271, 1272, 1, '.aws-test.com', 25, 1000, 90, 1, 1, 10, 20, 5, 'Automated Aurora Testing Cluster 1')"); + admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) VALUES (1273, 1274, 1, '.cluster2.aws.test', 25, 1000, 90, 0, 1, 10, 20, 5, 'Automated Aurora Testing Cluster 2')"); + admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) VALUES (1275, 1276, 1, '.aws-test.com', 25, 1000, 90, 0, 2, 10, 20, 5, 'Automated Aurora Testing Cluster 3')"); +#else admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) VALUES (1271, 1272, 1, '.aws-test.com', 25, 120, 90, 1, 1, 10, 20, 5, 'Automated Aurora Testing Cluster 1')"); admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) VALUES (1273, 1274, 1, '.cluster2.aws.test', 25, 120, 90, 0, 1, 10, 20, 5, 'Automated Aurora Testing Cluster 2')"); admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) VALUES (1275, 1276, 1, '.aws-test.com', 25, 120, 90, 0, 2, 10, 20, 5, 'Automated Aurora Testing Cluster 3')"); - //admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) VALUES (1275, 1276, 1, '.aws.3.test.com', 25, 120, 90, 0, 2, 10, 20, 5, 'Automated Aurora Testing Cluster 3')"); +#endif admindb->execute("UPDATE mysql_aws_aurora_hostgroups SET active=1"); - //admindb->execute("update mysql_servers set max_replication_lag=20"); +} + +void ProxySQL_Admin::enable_aurora_testing() { + proxy_info("Admin is enabling AWS Aurora Testing using SQLite3 Server and HGs from 1271 to 1276\n"); + mysql_servers_wrlock(); + enable_aurora_testing_populate_mysql_servers(); + enable_aurora_testing_populate_mysql_aurora_hostgroups(); load_mysql_servers_to_runtime(); mysql_servers_wrunlock(); - //admindb->execute("UPDATE global_variables SET variable_value=3000 WHERE variable_name='mysql-monitor_ping_interval'"); - //admindb->execute("UPDATE global_variables SET variable_value=1500 WHERE variable_name='mysql-monitor_ping_timeout'"); - //admindb->execute("UPDATE global_variables SET variable_value=3000 WHERE variable_name='mysql-monitor_replication_lag_interval'"); - //admindb->execute("UPDATE global_variables SET variable_value=1500 WHERE variable_name='mysql-monitor_replication_lag_timeout'"); - admindb->execute("UPDATE global_variables SET variable_value=200 WHERE variable_name='mysql-monitor_ping_interval'"); + admindb->execute("UPDATE global_variables SET variable_value=1000 WHERE variable_name='mysql-monitor_ping_interval'"); admindb->execute("UPDATE global_variables SET variable_value=3000 WHERE variable_name='mysql-monitor_ping_timeout'"); - admindb->execute("UPDATE global_variables SET variable_value=200 WHERE variable_name='mysql-monitor_replication_lag_interval'"); + admindb->execute("UPDATE global_variables SET variable_value=1000 WHERE variable_name='mysql-monitor_replication_lag_interval'"); admindb->execute("UPDATE global_variables SET variable_value=3000 WHERE variable_name='mysql-monitor_replication_lag_timeout'"); admindb->execute("UPDATE global_variables SET variable_value='percona.heartbeat' WHERE variable_name='mysql-monitor_replication_lag_use_percona_heartbeat'"); load_mysql_variables_to_runtime(); + admindb->execute("DELETE FROM mysql_users WHERE username LIKE '%aurora%'"); admindb->execute("INSERT INTO mysql_users (username,password,default_hostgroup) VALUES ('aurora1','pass1',1271), ('aurora2','pass2',1273), ('aurora3','pass3',1275)"); init_users(); admindb->execute("INSERT INTO mysql_query_rules (active, username, match_pattern, destination_hostgroup, apply) VALUES (1, 'aurora1', '^SELECT.*max_lag_ms', 1272, 1)"); diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index cdb7fd40a1..ece152d442 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -33,6 +33,8 @@ #include #include +using std::string; + #define SELECT_VERSION_COMMENT "select @@version_comment limit 1" #define SELECT_VERSION_COMMENT_LEN 32 #define SELECT_DB_USER "select DATABASE(), USER() limit 1" @@ -317,6 +319,44 @@ bool match_monitor_query(const std::string& monitor_query, const std::string& qu } #endif // TEST_GROUPREP +#ifdef TEST_AURORA + +using std::vector; + +using aurora_hg_info_t = std::tuple; +enum AURORA_HG_INFO { + WRITER_HG, + READER_HG, + DOMAIN_NAME +}; + +vector get_hgs_info(SQLite3DB* db) { + vector whgs {}; + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + GloAdmin->admindb->execute_statement( + "SELECT writer_hostgroup,reader_hostgroup,domain_name FROM mysql_aws_aurora_hostgroups", + &error, &cols, &affected_rows, &resultset + ); + + for (const SQLite3_row* r : resultset->rows) { + uint32_t writer_hg = atoi(r->fields[0]); + uint32_t reader_hg = atoi(r->fields[1]); + string domain_name { r->fields[2] }; + + whgs.push_back({writer_hg, reader_hg, domain_name}); + } + + return whgs; +} + + +#endif + void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { char *error=NULL; @@ -735,7 +775,34 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); - GloSQLite3Server->populate_aws_aurora_table(sess); + + if (strcasestr(query_no_space, TEST_AURORA_MONITOR_BASE_QUERY)) { + string s_whg { query_no_space + strlen(TEST_AURORA_MONITOR_BASE_QUERY) }; + uint32_t whg = atoi(s_whg.c_str()); + + GloSQLite3Server->populate_aws_aurora_table(sess, whg); + vector hgs_info { get_hgs_info(GloAdmin->admindb) }; + + const auto match_writer = [&whg](const aurora_hg_info_t& hg_info) { + return std::get(hg_info) == whg; + }; + const auto hg_info_it = std::find_if(hgs_info.begin(), hgs_info.end(), match_writer); + string select_query { + "SELECT SERVER_ID,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS,CPU" + " FROM REPLICA_HOST_STATUS " + }; + + if (hg_info_it == hgs_info.end()) { + select_query += " LIMIT 0"; + } else { + const string& domain_name { std::get(*hg_info_it) }; + select_query += " WHERE DOMAIN_NAME='" + domain_name + "' ORDER BY SERVER_ID"; + } + + free(query); + query = static_cast(malloc(select_query.length() + 1)); + strcpy(query, select_query.c_str()); + } } #endif // TEST_AURORA #ifdef TEST_GALERA @@ -843,10 +910,12 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { pthread_mutex_unlock(&GloSQLite3Server->aurora_mutex); +#ifdef TEST_AURORA_RANDOM if (rand() % 100 == 0) { // randomly add some latency on 1% of the traffic sleep(2); } +#endif } #endif // TEST_AURORA #ifdef TEST_GALERA @@ -1369,27 +1438,187 @@ void SQLite3_Server::populate_galera_table(MySQL_Session *sess) { #endif // TEST_GALERA #ifdef TEST_AURORA -void SQLite3_Server::populate_aws_aurora_table(MySQL_Session *sess) { - // this function needs to be called with lock on mutex aurora_mutex already acquired - sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); - sqlite3_stmt *statement=NULL; - //sqlite3 *mydb3=sessdb->get_db(); - int rc; - char *query=(char *)"INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5)"; - //rc=sqlite3_prepare_v2(mydb3, query, -1, &statement, 0); - rc = sessdb->prepare_v2(query, &statement); - ASSERT_SQLITE_OK(rc, sessdb); + +float get_rand_cpu() { + int cpu_i = rand() % 10000; + float cpu = static_cast(cpu_i) / 100; + + return cpu; +} + +string get_curtime_str() { time_t __timer; char lut[30]; struct tm __tm_info; time(&__timer); localtime_r(&__timer, &__tm_info); strftime(lut, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); + string s = string(lut); + return s; +} + +void bind_query_params( + SQLite3DB* db, + sqlite3_stmt* stmt, + const string& server_id, + const string& domain, + const string& session_id, + float cpu, + const string& lut, + int32_t lag_ms +) { + int rc = 0; + + rc=sqlite3_bind_text(stmt, 1, server_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc=sqlite3_bind_text(stmt, 2, domain.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc=sqlite3_bind_text(stmt, 3, session_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc=sqlite3_bind_double(stmt, 4, cpu); ASSERT_SQLITE_OK(rc, db); + rc=sqlite3_bind_text(stmt, 5, lut.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc=sqlite3_bind_double(stmt, 6, lag_ms); ASSERT_SQLITE_OK(rc, db); + SAFE_SQLITE3_STEP2(stmt); + rc=sqlite3_clear_bindings(stmt); ASSERT_SQLITE_OK(rc, db); + rc=sqlite3_reset(stmt); ASSERT_SQLITE_OK(rc, db); +} + +/** + * @brief Extracts SERVER_ID from the supplied hostname using DOMAIN_NAME. + * @param hostname The server hostname (SERVER_ID + DOMAIN_NAME)). + * @param domain_name The server DOMAIN_NAME as in 'mysql_aws_aurora_hostgroups' + * @return Either the SERVER_ID in the supplied hostname or empty if DOMAIN_NAME failed to match. + */ +string get_server_id(const string& hostname, const string& domain_name) { + string::size_type pos = hostname.find(domain_name); + + if (pos == string::npos) { + return {}; + } else { + return hostname.substr(0, pos); + } +} + +void SQLite3_Server::populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg) { + int rc = 0; + sqlite3_stmt* stmt = NULL; + const char query[] { "INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5, ?6)" }; + + rc = sessdb->prepare_v2(query, &stmt); + ASSERT_SQLITE_OK(rc, sessdb); + +#ifndef TEST_AURORA_RANDOM + SQLite3_result* host_status = NULL; + + { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + + string query { + "SELECT SERVER_ID,DOMAIN_NAME,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS" + " FROM REPLICA_HOST_STATUS" + }; + sessdb->execute_statement(query.c_str(), &error, &cols, &affected_rows, &host_status); + } + + // If empty, we fill the map with sensible defaults for performing manual testing. + if (host_status->rows.empty()) { + vector hgs_info { get_hgs_info(GloAdmin->admindb) }; + SQLite3_result* resultset = nullptr; + + { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + + GloAdmin->admindb->execute_statement( + "SELECT hostname, hostgroup_id FROM mysql_servers WHERE hostgroup_id BETWEEN 1270 AND 1300" + " GROUP BY HOSTNAME", + &error, &cols, &affected_rows, &resultset + ); + } + + sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); + vector proc_srvs {}; + + for (const aurora_hg_info_t& hg_info : hgs_info) { + const auto match_writer = [&hg_info](const SQLite3_row* row) { + return atoi(row->fields[1]) == std::get(hg_info); + }; + const auto mysrv_it = std::find_if(resultset->rows.begin(), resultset->rows.end(), match_writer); + bool writer_set = false; + + for (const SQLite3_row* r : resultset->rows) { + const string srv_hostname { r->fields[0] }; + const uint32_t srv_hg_id = atoi(r->fields[1]); + const string& aurora_domain { std::get(hg_info) }; + + if ( + srv_hostname.find(aurora_domain) == string::npos || + std::find(proc_srvs.begin(), proc_srvs.end(), srv_hostname) != proc_srvs.end() + ) { + continue; + } + + const string server_id { + get_server_id(srv_hostname, std::get(hg_info)) + }; + + string session_id {}; + + if ( + (mysrv_it == resultset->rows.end() && writer_set == false) || + (srv_hg_id == std::get(hg_info) && writer_set == false) + ) { + session_id = "MASTER_SESSION_ID"; + writer_set = true; + } else { + session_id = "TESTID-" + server_id + aurora_domain + "-R"; + } + + const float cpu = get_rand_cpu(); + const string lut { get_curtime_str() }; + const int lag_ms = 0; + + bind_query_params(sessdb, stmt, server_id, aurora_domain, session_id, cpu, lut, lag_ms); + proc_srvs.push_back(srv_hostname); + } + } + + sqlite3_finalize(stmt); + delete resultset; + } else { + // We just re-generate deterministic 'SESSION_IDS', preserving 'MASTER_SESSION_ID' values: + // 'SESSION_IDS' are preserved, 'MASTER_SESSION_ID' or others. + for (SQLite3_row* row : host_status->rows) { + const char* server_id = row->fields[0]; + const char* domain_name = row->fields[1]; + + const char update_query_t[] { + "UPDATE REPLICA_HOST_STATUS SET SESSION_ID='%s',CPU=%f,LAST_UPDATE_TIMESTAMP='%s'" + " WHERE SERVER_ID='%s' AND DOMAIN_NAME='%s' AND SESSION_ID!='MASTER_SESSION_ID'" + }; + + const string session_id { "TESTID-" + string { server_id } + domain_name + "-R" }; + const float cpu = get_rand_cpu(); + const string lut { get_curtime_str() }; + + const string update_query { + cstr_format(update_query_t, session_id.c_str(), cpu, lut.c_str(), server_id, domain_name).str + }; + + sessdb->execute(update_query.c_str()); + } + } + + delete host_status; +#else + sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); + + string lut { get_curtime_str() }; string myip = string(sess->client_myds->proxy_addr.addr); string clu_id_s = myip.substr(6,1); unsigned int cluster_id = atoi(clu_id_s.c_str()); cluster_id--; - //if (rand() % 200 == 0) { + if (rand() % 20000 == 0) { // simulate a failover cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id]; @@ -1410,14 +1639,12 @@ void SQLite3_Server::populate_aws_aurora_table(MySQL_Session *sess) { } } for (unsigned int i=0; i; insert_into_tables_defs(tables_defs_aurora, (const char *)"REPLICA_HOST_STATUS", - (const char *)"CREATE TABLE REPLICA_HOST_STATUS (SERVER_ID VARCHAR NOT NULL, SESSION_ID VARCHAR NOT NULL, CPU REAL NOT NULL, LAST_UPDATE_TIMESTAMP VARCHAR NOT NULL, REPLICA_LAG_IN_MILLISECONDS REAL NOT NULL)"); + "CREATE TABLE REPLICA_HOST_STATUS (" + " SERVER_ID VARCHAR NOT NULL , DOMAIN_NAME VARCHAR NOT NULL , SESSION_ID VARCHAR NOT NULL ," + " CPU REAL NOT NULL , LAST_UPDATE_TIMESTAMP VARCHAR NOT NULL , REPLICA_LAG_IN_MILLISECONDS REAL NOT NULL ," + " PRIMARY KEY (SERVER_ID, DOMAIN_NAME)" + ")" + ); check_and_build_standard_tables(sessdb, tables_defs_aurora); GloAdmin->enable_aurora_testing(); #endif // TEST_AURORA From efe83fea55b361549ea1e3abccd662f7c5f966a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 30 Jun 2023 13:47:00 +0200 Subject: [PATCH 41/56] Fix propagation of 'OFFLINE_HARD' servers due to 'read_only_action_v2' --- lib/MySQL_HostGroups_Manager.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 618b333f9e..9f911a23d9 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -4871,14 +4871,16 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listchecksum_variables.checksum_mysql_servers) { + // NOTE: We are always required to remove 'OFFLINE_HARD' servers, since we are not interested in + // their propagation to other cluster members. + unique_ptr runtime_servers_resultset { get_admin_runtime_mysql_servers(mydb) }; + remove_resultset_offline_hard_servers(runtime_servers_resultset); + save_runtime_mysql_servers(runtime_servers_resultset.release()); + char* error = NULL; int cols = 0; int affected_rows = 0; SQLite3_result* resultset = NULL; - mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); - save_runtime_mysql_servers(resultset); // assigning runtime_mysql_servers with updated mysql server resultset - - resultset = NULL; // reset mysql_server checksum table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = 0; From 97463ca96c3b58ea4fc74322eb4e62a8d6888cf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 30 Jun 2023 13:47:41 +0200 Subject: [PATCH 42/56] Fix invalid use of SHUNNED servers for checksum computation in 'read_only_action_v2' --- lib/MySQL_HostGroups_Manager.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 9f911a23d9..1e5a80013a 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -4886,8 +4886,7 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::list3 ORDER BY hostgroup_id, hostname, port"; - mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + mydb->execute_statement(MYSQL_SERVERS_CHECKSUM_QUERY, &error, &cols, &affected_rows, &resultset); if (resultset) { if (resultset->rows_count) { From 3fb6ab9146c91c6c10ac82d5891bfa2dae1e351b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 5 Jul 2023 13:19:50 +0200 Subject: [PATCH 43/56] Fix compilation on Centos 6 due to old GCC limitations with 'auto' --- lib/proxysql_utils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index 556d21cdd0..8f2841cee4 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -461,7 +461,9 @@ std::string get_checksum_from_hash(uint64_t hash) { void remove_sqlite3_resultset_rows( unique_ptr& resultset, const function& pred ) { - const auto remove_it { std::remove_if(resultset->rows.begin(), resultset->rows.end(), pred) }; + const vector::iterator remove_it { + std::remove_if(resultset->rows.begin(), resultset->rows.end(), pred) + }; resultset->rows.erase(remove_it, resultset->rows.end()); resultset->rows_count = resultset->rows.size(); } From 43d627406056fb30a52018925b94bd7c4454a29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 10 Jul 2023 14:34:39 +0200 Subject: [PATCH 44/56] Fix servers resultsets not updated for empty config If these resultsets are not updated, checksums will always mismatch between the peer cluster nodes in case servers are deleted. The primary will hold the correct checksum ('0x0'), but will report the old resultset to the peer node, the second will compute a checksum different than the expected '0x0'. Cycle won't stop until config is updated in the target primary. --- lib/MySQL_HostGroups_Manager.cpp | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 823dcab681..216288bfd3 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2306,12 +2306,8 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru resultset->rows.end() ); resultset->rows_count = init_row_count - rm_rows_count; - - save_runtime_mysql_servers(resultset); - } else { - delete resultset; - resultset = nullptr; } + save_runtime_mysql_servers(resultset); } else { proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); } @@ -2367,11 +2363,8 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_v2_checksum(SQLite3_result* ); resultset->rows_count = init_row_count - rm_rows_count; - save_mysql_servers_v2(resultset); - } else { - delete resultset; - resultset = nullptr; } + save_mysql_servers_v2(resultset); } else { proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_v2", (long unsigned int)0); } @@ -8335,4 +8328,4 @@ uint64_t MySQL_HostGroups_Manager::compute_mysql_servers_raw_checksum(const SQLi myhash.Final(&res_hash, &hash2); return res_hash; -} \ No newline at end of file +} From f48dfce6cf91e73ac4ee2ae0e39c70dd0c950205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 11 Jul 2023 17:54:16 +0200 Subject: [PATCH 45/56] Simplified resultset and checksum generation for 'mysql_servers' - Resultsets for 'runtime_mysql_servers' and 'mysql_servers_v2' are now directly generated by the SQL query. - Checksum computations are simplified to 'SQLite3_result::raw_checksum' and equivalent function 'mysql_raw_checksum'. --- include/MySQL_HostGroups_Manager.h | 68 +++++++++++++++-- lib/MySQL_HostGroups_Manager.cpp | 115 ++--------------------------- lib/ProxySQL_Admin.cpp | 23 +----- lib/ProxySQL_Cluster.cpp | 70 +----------------- 4 files changed, 76 insertions(+), 200 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 3035677a78..d674c58b58 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -66,8 +66,55 @@ using json = nlohmann::json; #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" -//#define MYHGM_GEN_INCOMING_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming ORDER BY hostgroup_id, hostname, port" -#define MYHGM_GEN_ADMIN_MYSQL_SERVERS "SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port" +/* + * @brief Generates the 'runtime_mysql_servers' resultset exposed to other ProxySQL cluster members. + * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states + * are by definition local transitory states, this is why a 'mysql_servers' table reconfiguration isn't + * normally performed when servers are internally imposed with these statuses. This means, that propagating + * this state to other cluster members is undesired behavior, and so it's generating a different checksum, + * due to a server having this particular state, that will result in extra unnecessary fetching operations. + * The query also filters out 'OFFLINE_HARD' servers, 'OFFLINE_HARD' is a local status which is equivalent to + * a server no longer being part of the table (DELETED state). And so, they shouldn't be propagated. + * + * For placing the query into a single line for debugging purposes: + * ``` + * sed 's/^\t\+"//g; s/"\s\\$//g; s/\\"/"/g' /tmp/select.sql | paste -sd '' + * ``` + */ +#define MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS \ + "SELECT " \ + "hostgroup_id, hostname, port, gtid_port," \ + "CASE status" \ + " WHEN 0 THEN \"ONLINE\"" \ + " WHEN 1 THEN \"ONLINE\"" \ + " WHEN 2 THEN \"OFFLINE_SOFT\"" \ + " WHEN 3 THEN \"OFFLINE_HARD\"" \ + " WHEN 4 THEN \"ONLINE\" " \ + "END status," \ + "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ + "FROM mysql_servers " \ + "WHERE status != 3 " \ + "ORDER BY hostgroup_id, hostname, port" \ + +/** + * @brief Generates the 'mysql_servers_v2' resultset exposed to other ProxySQL cluster members. + * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config + * ('mysql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin + * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also + * filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration + * statuses, they are local, transient status that ProxySQL uses during operation. + */ +#define MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS \ + "SELECT " \ + "hostgroup_id, hostname, port, gtid_port, " \ + "CASE" \ + " WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \ + " ELSE status " \ + "END AS status, " \ + "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ + "FROM main.mysql_servers " \ + "WHERE status != \"OFFLINE_HARD\" " \ + "ORDER BY hostgroup_id, hostname, port" \ typedef std::unordered_map umap_mysql_errors; @@ -785,7 +832,20 @@ class MySQL_HostGroups_Manager { SQLite3_result* get_current_mysql_table(const string& name); SQLite3_result * execute_query(char *query, char **error); - SQLite3_result *dump_table_mysql(const string&); + /** + * @brief Creates a resultset with the current full content of the target table. + * @param string The target table. Valid values are: + * - "mysql_aws_aurora_hostgroups" + * - "mysql_galera_hostgroups" + * - "mysql_group_replication_hostgroups" + * - "mysql_replication_hostgroups" + * - "mysql_hostgroup_attributes" + * - "mysql_servers" + * - "cluster_mysql_servers" + * When targeting 'mysql_servers' table is purged and regenerated. + * @return The generated resultset. + */ + SQLite3_result* dump_table_mysql(const string&); /** * @brief Update the public member resulset 'mysql_servers_to_monitor'. This resulset should contain the latest @@ -882,8 +942,6 @@ class MySQL_HostGroups_Manager { MySrvC* find_server_in_hg(unsigned int _hid, const std::string& addr, int port); private: - static uint64_t compute_mysql_servers_raw_checksum(const SQLite3_result* runtime_mysql_servers); - void update_hostgroup_manager_mappings(); uint64_t get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers = nullptr); uint64_t get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2 = nullptr); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 216288bfd3..f9f7791de2 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2285,28 +2285,9 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru int cols = 0; int affected_rows = 0; - mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + mydb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); if (resultset) { - if (resultset->rows_count) { - // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster - // nodes, or relevant for checksum computation. - const size_t init_row_count = resultset->rows_count; - size_t rm_rows_count = 0; - const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - rm_rows_count += 1; - return true; - } else { - return false; - } - }; - resultset->rows.erase( - std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), - resultset->rows.end() - ); - resultset->rows_count = init_row_count - rm_rows_count; - } save_runtime_mysql_servers(resultset); } else { proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); @@ -2316,7 +2297,7 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru save_runtime_mysql_servers(runtime_mysql_servers); } - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = compute_mysql_servers_raw_checksum(resultset); + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = resultset != nullptr ? resultset->raw_checksum() : 0; proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]); return table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; @@ -2342,28 +2323,9 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_v2_checksum(SQLite3_result* int cols = 0; int affected_rows = 0; - GloAdmin->admindb->execute_statement(MYHGM_GEN_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); + GloAdmin->admindb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); if (resultset) { - if (resultset->rows_count) { - - const size_t init_row_count = resultset->rows_count; - size_t rm_rows_count = 0; - const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - rm_rows_count += 1; - return true; - } else { - return false; - } - }; - resultset->rows.erase( - std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), - resultset->rows.end() - ); - resultset->rows_count = init_row_count - rm_rows_count; - - } save_mysql_servers_v2(resultset); } else { proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_v2", (long unsigned int)0); @@ -2380,7 +2342,7 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_v2_checksum(SQLite3_result* table_resultset_checksum[MYSQL_AWS_AURORA_HOSTGROUPS] = 0; table_resultset_checksum[MYSQL_HOSTGROUP_ATTRIBUTES] = 0; - table_resultset_checksum[MYSQL_SERVERS_V2] = compute_mysql_servers_raw_checksum(resultset); + table_resultset_checksum[MYSQL_SERVERS_V2] = resultset != nullptr ? resultset->raw_checksum() : 0; proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_v2", table_resultset_checksum[MYSQL_SERVERS_V2]); commit_update_checksums_from_tables(); @@ -2988,7 +2950,9 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) } else if (name == "mysql_hostgroup_attributes") { query=(char *)"SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, servers_defaults, comment FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; } else if (name == "mysql_servers") { - query = (char *)MYHGM_GEN_ADMIN_RUNTIME_SERVERS; + query = (char *)MYHGM_GEN_ADMIN_RUNTIME_SERVERS; + } else if (name == "cluster_mysql_servers") { + query = (char *)MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS; } else { assert(0); } @@ -4293,7 +4257,7 @@ SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& return this->incoming_replication_hostgroups; } else if (name == "mysql_hostgroup_attributes") { return this->incoming_hostgroup_attributes; - } else if (name == "mysql_servers") { + } else if (name == "cluster_mysql_servers") { return this->runtime_mysql_servers; } else if (name == "mysql_servers_v2") { return this->incoming_mysql_servers_v2; @@ -8266,66 +8230,3 @@ void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::remove_HGM(MySrvC* srv) srv->status = MYSQL_SERVER_STATUS_OFFLINE_HARD; srv->ConnectionsFree->drop_all_connections(); } - -/** - * @brief This function computes the checksum of the mysql_servers resultset. -* As the checksum is being calculated, the function replaces the status values with their respective integer values. - * - * @param mysql_servers resultset of mysql_servers or mysql_servers_incoming. - */ -uint64_t MySQL_HostGroups_Manager::compute_mysql_servers_raw_checksum(const SQLite3_result* mysql_servers) { - - if (!mysql_servers || mysql_servers->rows_count == 0) - return 0; - - int status_idx = -1; - - for (int i = 0; i < mysql_servers->columns; i++) { - if (mysql_servers->column_definition[i] && mysql_servers->column_definition[i]->name && - strcmp(mysql_servers->column_definition[i]->name, "status") == 0) { - status_idx = i; - break; - } - } - - if (status_idx == -1) assert(0); - - SpookyHash myhash; - myhash.Init(19, 3); - - for (const SQLite3_row* r : mysql_servers->rows) { - - const char* mapped_status = ""; - const char* status = r->fields[status_idx]; - - if (status) { - if (strcasecmp(status, "OFFLINE_HARD") == 0) - continue; - - if (strcasecmp(status, "ONLINE") == 0 || - strcasecmp(status, "SHUNNED") == 0) { - mapped_status = "0"; - } else if (strcasecmp(status, "OFFLINE_SOFT") == 0) { - mapped_status = "2"; - } - } - - for (int i = 0; i < mysql_servers->columns; i++) { - - if (r->fields[i]) { - if (i != status_idx) { - myhash.Update(r->fields[i], r->sizes[i]); - } else { - myhash.Update(mapped_status, strlen(mapped_status)); - } - } else { - myhash.Update("", 0); - } - } - } - - uint64_t res_hash = 0, hash2 = 0; - myhash.Final(&res_hash, &hash2); - - return res_hash; -} diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f830b9ddae..629208d628 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -3759,7 +3759,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats string tn = ""; if (!strncasecmp(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS))) { - tn = "mysql_servers"; + tn = "cluster_mysql_servers"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) { tn = "mysql_replication_hostgroups"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS))) { @@ -3787,29 +3787,10 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { char *error=NULL; int cols=0; int affected_rows=0; - const char* query = "SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); GloAdmin->mysql_servers_wrlock(); - GloAdmin->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + GloAdmin->admindb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); GloAdmin->mysql_servers_wrunlock(); - if (resultset && resultset->rows_count) { - const size_t init_row_count = resultset->rows_count; - size_t rm_rows_count = 0; - const auto is_offline_server = [&rm_rows_count](SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - rm_rows_count += 1; - return true; - } else { - return false; - } - }; - resultset->rows.erase( - std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), - resultset->rows.end() - ); - - resultset->rows_count = init_row_count - rm_rows_count; - } } else { resultset = MyHGM->dump_table_mysql(tn); } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index a909683629..71a033e46c 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1683,64 +1683,6 @@ int ProxySQL_Cluster::fetch_and_store(MYSQL* conn, const fetch_query& f_query, M return query_res; } -/** - * @brief Generates a hash for the resulset received for the query 'CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS'. - * @details Remember that this query query is intercepted by 'ProxySQL_Admin' and always answered via - * 'MySQL_HostGroups_Manager::dump_table_proxysql_servers'. - * @param resultset The resulset resulting from the mentioned query. - * @return A hash representing the contents of the resulset. - */ -uint64_t mysql_servers_raw_checksum(MYSQL_RES* resultset) { - if (resultset == nullptr) { return 0; } - - uint64_t num_rows = mysql_num_rows(resultset); - if (num_rows == 0) { return 0; } - - MYSQL_FIELD* fields = mysql_fetch_fields(resultset); - uint32_t num_fields = mysql_num_fields(resultset); - uint32_t status_idx = 0; - - for (uint32_t i = 0; i < num_fields; i++) { - if (strcmp(fields[i].name, "status") == 0) { - status_idx = i; - } - } - - SpookyHash myhash {}; - myhash.Init(19,3); - - while (MYSQL_ROW row = mysql_fetch_row(resultset)) { - for (uint32_t i = 0; i < num_fields; i++) { - if (strcmp(row[status_idx], "OFFLINE_HARD") == 0) { - continue; - } - - if (row[i]) { - if (strcmp(fields[i].name, "status") == 0) { - if (strcmp(row[i], "ONLINE") == 0 || strcmp(row[i], "SHUNNED") == 0) { - myhash.Update("0", strlen("0")); - } else { - myhash.Update("2", strlen("1")); - } - } else { - // computing 'strlen' is required see @details - myhash.Update(row[i], strlen(row[i])); - } - } else { - myhash.Update("", 0); - } - } - } - - // restore the initial resulset index - mysql_data_seek(resultset, 0); - - uint64_t res_hash = 0, hash2 = 0; - myhash.Final(&res_hash, &hash2); - - return res_hash; -} - /** * @brief Generates a hash from the received resultsets from executing the following queries in the specified * order: @@ -1761,13 +1703,7 @@ uint64_t compute_servers_tables_raw_checksum(const vector& results, SpookyHash myhash {}; for (size_t i = 0; i < size; i++) { - uint64_t raw_hash = 0; - - if (i == 0) { - raw_hash = mysql_servers_raw_checksum(results[i]); - } else { - raw_hash = mysql_raw_checksum(results[i]); - } + uint64_t raw_hash = mysql_raw_checksum(results[i]); if (raw_hash != 0) { if (init == false) { @@ -1871,7 +1807,7 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ } if (result != nullptr) { - const uint64_t servers_hash = mysql_servers_raw_checksum(result); + const uint64_t servers_hash = mysql_raw_checksum(result); const string computed_checksum{ get_checksum_from_hash(servers_hash) }; proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); @@ -2116,7 +2052,7 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch bool runtime_checksum_matches = true; if (results[6]) { - const uint64_t runtime_mysql_server_hash = mysql_servers_raw_checksum(results[6]); + const uint64_t runtime_mysql_server_hash = mysql_raw_checksum(results[6]); const std::string runtime_mysql_server_computed_checksum = get_checksum_from_hash(runtime_mysql_server_hash); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); From 253e309aa02283d3452773b53808244d26c67118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 11 Jul 2023 18:01:51 +0200 Subject: [PATCH 46/56] Cleanup left commented function 'update_runtime_mysql_servers_table' --- include/MySQL_HostGroups_Manager.h | 1 - lib/MySQL_HostGroups_Manager.cpp | 229 ----------------------------- 2 files changed, 230 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index d674c58b58..cf42a6dfc6 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -800,7 +800,6 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); - //void update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server); bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, SQLite3_result* mysql_servers_v2 = nullptr, const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {}, bool only_commit_runtime_mysql_servers = false, bool update_version = false); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f9f7791de2..a2da060226 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1498,235 +1498,6 @@ unsigned int MySQL_HostGroups_Manager::get_servers_table_version() { return __sync_fetch_and_add(&status.servers_table_version,0); } -///** -// * @brief Generates runtime mysql server records and checksum. -// * -// * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in -// * the specified order. -// * @param can be null or previously generated runtime_mysql_servers resultset can be passed. -// */ -//void MySQL_HostGroups_Manager::update_runtime_mysql_servers_table(SQLite3_result* runtime_mysql_servers, -// const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server) { -// char* error = NULL; -// int cols = 0; -// int affected_rows = 0; -// SQLite3_result* resultset = NULL; -// // if any server has gtid_port enabled, use_gtid is set to true -// // and then has_gtid_port is set too -// bool use_gtid = false; -// -// mydb->execute("DELETE FROM mysql_servers"); -// generate_mysql_servers_table(); -// -// const char* query = "SELECT mem_pointer, t1.hostgroup_id, t1.hostname, t1.port FROM mysql_servers t1 LEFT OUTER JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE t2.hostgroup_id IS NULL"; -// mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); -// if (error) { -// proxy_error("Error on %s : %s\n", query, error); -// } else { -// if (GloMTH->variables.hostgroup_manager_verbose) { -// proxy_info("Dumping mysql_servers LEFT JOIN mysql_servers_incoming\n"); -// resultset->dump_to_stderr(); -// } -// -// for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { -// SQLite3_row* r = *it; -// long long ptr = atoll(r->fields[0]); -// proxy_warning("Removed server at address %lld, hostgroup %s, address %s port %s. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", ptr, r->fields[1], r->fields[2], r->fields[3]); -// MySrvC* mysrvc = (MySrvC*)ptr; -// mysrvc->status = MYSQL_SERVER_STATUS_OFFLINE_HARD; -// mysrvc->ConnectionsFree->drop_all_connections(); -// char* q1 = (char*)"DELETE FROM mysql_servers WHERE mem_pointer=%lld"; -// char* q2 = (char*)malloc(strlen(q1) + 32); -// sprintf(q2, q1, ptr); -// mydb->execute(q2); -// free(q2); -// } -// } -// if (resultset) { delete resultset; resultset = NULL; } -// -// mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); -// -// // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) -// query = (char*)"SELECT t1.*, t2.gtid_port, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.gtid_port<>t2.gtid_port OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); -// mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); -// if (error) { -// proxy_error("Error on %s : %s\n", query, error); -// } else { -// -// if (GloMTH->variables.hostgroup_manager_verbose) { -// proxy_info("Dumping mysql_servers JOIN mysql_servers_incoming\n"); -// resultset->dump_to_stderr(); -// } -// // optimization #829 -// int rc; -// sqlite3_stmt* statement1 = NULL; -// sqlite3_stmt* statement2 = NULL; -// //sqlite3 *mydb3=mydb->get_db(); -// char* query1 = (char*)"UPDATE mysql_servers SET mem_pointer = ?1 WHERE hostgroup_id = ?2 AND hostname = ?3 AND port = ?4"; -// //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); -// rc = mydb->prepare_v2(query1, &statement1); -// ASSERT_SQLITE_OK(rc, mydb); -// char* query2 = (char*)"UPDATE mysql_servers SET weight = ?1 , status = ?2 , compression = ?3 , max_connections = ?4 , max_replication_lag = ?5 , use_ssl = ?6 , max_latency_ms = ?7 , comment = ?8 , gtid_port = ?9 WHERE hostgroup_id = ?10 AND hostname = ?11 AND port = ?12"; -// //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query2, -1, &statement2, 0); -// rc = mydb->prepare_v2(query2, &statement2); -// ASSERT_SQLITE_OK(rc, mydb); -// -// for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { -// SQLite3_row* r = *it; -// long long ptr = atoll(r->fields[12]); // increase this index every time a new column is added -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), ptr, atoi(r->fields[0]), atoi(r->fields[6])); -// //fprintf(stderr,"%lld\n", ptr); -// if (ptr == 0) { -// if (GloMTH->variables.hostgroup_manager_verbose) { -// proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5])); -// } -// MySrvC* mysrvc = new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), atoi(r->fields[10]), r->fields[11]); // add new fields here if adding more columns in mysql_servers -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), mysrvc, atoi(r->fields[0])); -// MyHGM->add(mysrvc, atoi(r->fields[0])); -// ptr = (uintptr_t)mysrvc; -// rc = (*proxy_sqlite3_bind_int64)(statement1, 1, ptr); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_text)(statement1, 3, r->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, mydb); -// SAFE_SQLITE3_STEP2(statement1); -// rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); -// if (mysrvc->gtid_port) { -// // this server has gtid_port configured, we set use_gtid -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); -// use_gtid = true; -// } -// } else { -// bool run_update = false; -// MySrvC* mysrvc = (MySrvC*)ptr; -// // carefully increase the 2nd index by 1 for every new column added -// if (atoi(r->fields[3]) != atoi(r->fields[13])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing gtid_port for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), mysrvc->gtid_port, atoi(r->fields[13])); -// mysrvc->gtid_port = atoi(r->fields[13]); -// } -// -// if (atoi(r->fields[4]) != atoi(r->fields[14])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), mysrvc->weight, atoi(r->fields[14])); -// mysrvc->weight = atoi(r->fields[14]); -// } -// if (atoi(r->fields[5]) != atoi(r->fields[15])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing status for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[5]), mysrvc->status, atoi(r->fields[15])); -// mysrvc->status = (MySerStatus)atoi(r->fields[15]); -// if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED) { -// mysrvc->shunned_automatic = false; -// } -// } -// if (atoi(r->fields[6]) != atoi(r->fields[16])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing compression for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[6]), mysrvc->compression, atoi(r->fields[16])); -// mysrvc->compression = atoi(r->fields[16]); -// } -// if (atoi(r->fields[7]) != atoi(r->fields[17])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing max_connections for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[7]), mysrvc->max_connections, atoi(r->fields[17])); -// mysrvc->max_connections = atoi(r->fields[17]); -// } -// if (atoi(r->fields[8]) != atoi(r->fields[18])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing max_replication_lag for server %u:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[8]), mysrvc->max_replication_lag, atoi(r->fields[18])); -// mysrvc->max_replication_lag = atoi(r->fields[18]); -// if (mysrvc->max_replication_lag == 0) { // we just changed it to 0 -// if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { -// // the server is currently shunned due to replication lag -// // but we reset max_replication_lag to 0 -// // therefore we immediately reset the status too -// mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; -// } -// } -// } -// if (atoi(r->fields[9]) != atoi(r->fields[19])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing use_ssl for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[9]), mysrvc->use_ssl, atoi(r->fields[19])); -// mysrvc->use_ssl = atoi(r->fields[19]); -// } -// if (atoi(r->fields[10]) != atoi(r->fields[20])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing max_latency_ms for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[10]), mysrvc->max_latency_us / 1000, atoi(r->fields[20])); -// mysrvc->max_latency_us = 1000 * atoi(r->fields[20]); -// } -// if (strcmp(r->fields[11], r->fields[21])) { -// if (GloMTH->variables.hostgroup_manager_verbose) -// proxy_info("Changing comment for server %d:%s:%d (%s:%d) from '%s' to '%s'\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[11], r->fields[21]); -// free(mysrvc->comment); -// mysrvc->comment = strdup(r->fields[21]); -// } -// if (run_update) { -// rc = (*proxy_sqlite3_bind_int64)(statement2, 1, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 2, mysrvc->status); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 3, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 4, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 5, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 6, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 7, mysrvc->max_latency_us / 1000); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_text)(statement2, 8, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 9, mysrvc->gtid_port); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 10, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_text)(statement2, 11, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_bind_int64)(statement2, 12, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); -// SAFE_SQLITE3_STEP2(statement2); -// rc = (*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, mydb); -// rc = (*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, mydb); -// } -// if (mysrvc->gtid_port) { -// // this server has gtid_port configured, we set use_gtid -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port); -// use_gtid = true; -// } -// } -// } -// (*proxy_sqlite3_finalize)(statement1); -// (*proxy_sqlite3_finalize)(statement2); -// } -// if (use_gtid) { -// has_gtid_port = true; -// } else { -// has_gtid_port = false; -// } -// if (resultset) { delete resultset; resultset = NULL; } -// -// purge_mysql_servers_table(); -// proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); -// mydb->execute("DELETE FROM mysql_servers"); -// generate_mysql_servers_table(); -// -// const auto mysql_servers_checksum = get_mysql_servers_checksum(); -// -// char buf[80]; -// uint32_t d32[2]; -// memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); -// sprintf(buf, "0x%0X%0X", d32[0], d32[1]); -// pthread_mutex_lock(&GloVars.checksum_mutex); -// GloVars.checksums_values.mysql_servers.set_checksum(buf); -// GloVars.checksums_values.mysql_servers.version++; -// //struct timespec ts; -// //clock_gettime(CLOCK_REALTIME, &ts); -// time_t t = time(NULL); -// -// if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && -// GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { -// GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; -// } else { -// GloVars.checksums_values.mysql_servers.epoch = t; -// } -// -// GloVars.checksums_values.updates_cnt++; -// GloVars.generate_global_checksum(); -// GloVars.epoch_version = t; -// pthread_mutex_unlock(&GloVars.checksum_mutex); -// -// update_hostgroup_manager_mappings(); -// update_table_mysql_servers_for_monitor(false); -//} - // we always assume that the calling thread has acquired a rdlock() int MySQL_HostGroups_Manager::servers_add(SQLite3_result *resultset) { if (resultset==NULL) { From fa71a5cce3dba0a0b3e50cdcfa622f4a581247de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 11 Jul 2023 18:36:19 +0200 Subject: [PATCH 47/56] Improve doc for 'mysql_servers_sync_algorithm' --- include/ProxySQL_Cluster.hpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index a93fa13f3c..feb7a71ed4 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -372,10 +372,29 @@ struct variable_type { }; }; +/** + * @brief Specifies the sync algorithm to use when pulling config from another ProxySQL cluster peer. + */ enum class mysql_servers_sync_algorithm { - runtime_mysql_servers_and_mysql_servers_v2 = 1, // sync runtime_mysql_servers and mysql_server_v2 from remote peer - mysql_servers_v2 = 2, // sync mysql_server_v2 from remote peer - auto_select = 3 // based on -M flag + /** + * @brief Sync 'runtime_mysql_servers' and 'mysql_server_v2' from remote peer. + * @details This means that both config and runtime servers info are to be synced, in other words, both + * user promoted config and runtime changes performed by ProxySQL ('Monitor') in the remote peer will + * trigger the start of syncing operations. + */ + runtime_mysql_servers_and_mysql_servers_v2 = 1, + /** + * @brief Sync only mysql_server_v2 (config) from the remote peer. + * @details Since 'runtime_mysql_servers' isn't considered for fetching, only config changes promoted by + * the user in the remote peer will by acknowledge and trigger the start of a syncing operation. + */ + mysql_servers_v2 = 2, + /** + * @brief Dependent on whether ProxySQL has been started with the -M flag. + * @details If '-M' is not present, 'runtime_mysql_servers_and_mysql_servers_v2' is selected, otherwise + * 'mysql_servers_v2' is chosen. + */ + auto_select = 3 }; /** From 664fba97d4a7bdb83acc06ccf64646ccfedc7752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 11 Jul 2023 18:37:10 +0200 Subject: [PATCH 48/56] Improve doc for 'pull_mysql_servers_v2_from_peer' --- lib/ProxySQL_Cluster.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 71a033e46c..221031cb2c 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1884,6 +1884,11 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ * fetching of different tables (mysql_servers vs mysql_servers_v2) and computing of checksum. In the previous version, * the checksum for "mysql_servers" was computed and added to the checksums of other dependent modules. In contrast, the new version * calculates the checksum for "mysql_servers_v2" and combines it with the checksums of other dependent modules. + * + * IMPORTANT: This function performs both the fetching of config, and conditionally the 'runtime_mysql_servers', in + * order to avoid extra transitory states and checksums that would result if this operation was performed in multiple + * steps. When required by the sync algorithm ('mysql_servers_sync_algorithm'), these two fetches and configuration + * promotion should be performed in a single 'atomic' operation. * * @param peer_mysql_server_v2 checksum and epoch of mysql_servers_v2 from remote peer * @param peer_runtime_mysql_server checksum and epoch of mysql_servers from remote peer From 5f53c185d07c3cb0fae49d690fe06c2641dfba7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 11 Jul 2023 21:27:51 +0200 Subject: [PATCH 49/56] Remove outdated comments and minor indentation fixes --- lib/MySQL_HostGroups_Manager.cpp | 7 ------- lib/ProxySQL_Cluster.cpp | 12 +++++------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index a2da060226..22ddd6ac6a 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1758,8 +1758,6 @@ bool MySQL_HostGroups_Manager::commit( //mydb->execute("DELETE FROM mysql_servers"); //generate_mysql_servers_table(); - // INSERT OR IGNORE INTO mysql_servers SELECT ... FROM mysql_servers_incoming - // proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status, compression, max_connections) SELECT hostgroup_id, hostname, port, weight, status, compression, max_connections FROM mysql_servers_incoming\n"); mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) @@ -1971,8 +1969,6 @@ bool MySQL_HostGroups_Manager::commit( if (update_version) GloVars.checksums_values.mysql_servers.version++; - //struct timespec ts; - //clock_gettime(CLOCK_REALTIME, &ts); if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; @@ -4901,9 +4897,6 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listepoch > own_epoch) // epoch is newer - ) { + ) { if (v->diff_check >= diff_ms) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with mysql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - + ProxySQL_Checksum_Value_2* runtime_mysql_server_checksum = &checksums_values.mysql_servers; const bool fetch_runtime = (mysql_server_sync_algo == mysql_servers_sync_algorithm::runtime_mysql_servers_and_mysql_servers_v2); @@ -964,9 +964,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { (own_version == 1) // we just booted || (v->epoch > own_epoch) // epoch is newer - //|| - //(v->checksum != own_checksum) // as runtime mysql server is generated on node itself, epoch can be newer so we also check checksum - ) { + ) { if (v->diff_check >= diff_ms) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); proxy_info("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); @@ -1788,7 +1786,7 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ string_format("Cluster: Fetching 'MySQL Servers' from peer %s:%d completed\n", fetch_servers_done, hostname, port); std::string fetch_servers_err; string_format("Cluster: Fetching 'MySQL Servers' from peer %s:%d failed: \n", fetch_servers_err, hostname, port); - + // Create fetching query fetch_query query = { CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS, @@ -1805,7 +1803,7 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ result = nullptr; } } - + if (result != nullptr) { const uint64_t servers_hash = mysql_raw_checksum(result); const string computed_checksum{ get_checksum_from_hash(servers_hash) }; From 8dab09ec11f03e42f9f746f70295ba1bb918b17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 13 Jul 2023 13:28:59 +0200 Subject: [PATCH 50/56] Fix invalid propagation of 'main.mysql_servers' config by 'commit' Config should only be used for checksum computation and cluster resultset generation for 'LOAD MYSQL SERVERS TO RUNTIME'. Otherwise user config ('main.mysql_servers') would be promoted at each 'commit' call, and propagated through the cluster members. --- include/MySQL_HostGroups_Manager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index cf42a6dfc6..e7f617ec5d 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -802,7 +802,7 @@ class MySQL_HostGroups_Manager { int servers_add(SQLite3_result *resultset); bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, SQLite3_result* mysql_servers_v2 = nullptr, const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {}, - bool only_commit_runtime_mysql_servers = false, bool update_version = false); + bool only_commit_runtime_mysql_servers = true, bool update_version = false); void commit_update_checksums_from_tables(); void CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() From 3b8e23e75ad6d7972863d2113f98b518b870a738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 14 Jul 2023 18:23:23 +0200 Subject: [PATCH 51/56] Fix invalid fallback response for 'CLUSTER_QUERY_MYSQL_SERVERS_V2' If 'MySQL_HostGroups_Manager::incoming_mysql_servers' happens to be empty when a 'CLUSTER_QUERY_MYSQL_SERVERS_V2' request is received in Admin, since there wasn't any user config promoted to runtime yet, the response should be an empty resulset. --- lib/ProxySQL_Admin.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 030189e7ba..cb3824caae 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -3789,17 +3789,23 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { GloAdmin->mysql_servers_wrunlock(); if (resultset == nullptr) { - - // This section of the code contains specific instructions for mysql_servers_v2, which is a virtual table - // that represents the mysql_servers (admin) records. In order to generate the resultset, data will be retrieved - // from mysql_server (admin) instead. + // 'mysql_servers_v2' is a virtual table that represents the latest 'main.mysql_servers' + // records promoted by the user. This section shouldn't be reached, since the initial resulset + // for this table ('MySQL_HostGroups_Manager::incoming_mysql_servers') is generated during + // initialization, and it's only updated in subsequent user config promotions. In case we + // reach here, an empty resultset should be replied, as it would mean that no user + // config has ever been promoted to runtime, and thus, this virtual table should remain empty. if (tn == "mysql_servers_v2") { + const string query_empty_resultset { + string { MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS } + " LIMIT 0" + }; + char *error=NULL; int cols=0; int affected_rows=0; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); GloAdmin->mysql_servers_wrlock(); - GloAdmin->admindb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); + GloAdmin->admindb->execute_statement(query_empty_resultset.c_str(), &error, &cols, &affected_rows, &resultset); GloAdmin->mysql_servers_wrunlock(); } else { resultset = MyHGM->dump_table_mysql(tn); From a4b6bd0b8fdc411ebee2dafed3eebed30093ce59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 19 Jul 2023 11:00:28 +0200 Subject: [PATCH 52/56] Refactor and homogenization of checksums computation - Simplified checksum computation for monitoring actions - Refactored checksum computation for 'mysql_servers' and 'mysql_servers_v2' modules in 'commit'. --- include/MySQL_HostGroups_Manager.h | 66 +++- include/proxysql_admin.h | 24 +- include/proxysql_utils.h | 9 - lib/MySQL_HostGroups_Manager.cpp | 468 +++++++++++------------------ lib/ProxySQL_Admin.cpp | 24 +- lib/ProxySQL_Cluster.cpp | 5 +- lib/proxysql_utils.cpp | 8 - 7 files changed, 278 insertions(+), 326 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 13eab752ae..365d9329f3 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -850,16 +850,64 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); - std::string gen_global_mysql_servers_checksum(); - bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, - SQLite3_result* mysql_servers_v2 = nullptr, const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {}, - bool only_commit_runtime_mysql_servers = true, bool update_version = false); - void commit_generate_mysql_servers_table(SQLite3_result* runtime_mysql_servers = nullptr); - void commit_update_checksum_from_mysql_servers(SpookyHash& myhash, bool& init); + /** + * @brief Generates a new global checksum for module 'mysql_servers_v2' using the provided hash. + * @param servers_v2_hash The 'raw_checksum' from 'MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS' or peer node. + * @return Checksum computed using the provided hash, and 'mysql_servers' config tables hashes. + */ + std::string gen_global_mysql_servers_v2_checksum(uint64_t servers_v2_hash); + bool commit( + const peer_runtime_mysql_servers_t& peer_runtime_mysql_servers = {}, + const peer_mysql_servers_v2_t& peer_mysql_servers_v2 = {}, + bool only_commit_runtime_mysql_servers = true, + bool update_version = false + ); + /** + * @brief Extracted from 'commit'. Performs the following actions: + * 1. Re-generates the 'myhgm.mysql_servers' table. + * 2. If supplied 'runtime_mysql_servers' is 'nullptr': + * 1. Gets the contents of the table via 'MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS'. + * 2. Save the resultset into 'this->runtime_mysql_servers'. + * 3. If supplied 'runtime_mysql_servers' isn't 'nullptr': + * 1. Updates the 'this->runtime_mysql_servers' with it. + * 4. Updates 'HGM_TABLES::MYSQL_SERVERS' with raw checksum from 'this->runtime_mysql_servers'. + * @param runtime_mysql_servers If not 'nullptr', used to update 'this->runtime_mysql_servers'. + * @return The updated 'MySQL_HostGroups_Manager::runtime_mysql_servers'. + */ + uint64_t commit_update_checksum_from_mysql_servers(SQLite3_result* runtime_mysql_servers = nullptr); + /** + * @brief Analogous to 'commit_generate_mysql_servers_table' but for 'incoming_mysql_servers_v2'. + */ + uint64_t commit_update_checksum_from_mysql_servers_v2(SQLite3_result* incoming_mysql_servers_v2 = nullptr); + /** + * @brief Update all HGM_TABLES checksums and uses them to update the supplied SpookyHash. + * @details Checksums are the checksums for the following tables: + * - mysql_replication_hostgroups + * - mysql_group_replication_hostgroups + * - mysql_galera_hostgroups + * - mysql_aws_aurora_hostgroups + * - mysql_hostgroup_attributes + * + * These checksums are used to compute the global checksum for 'mysql_servers_v2'. + * @param myhash SpookyHash to be updated with all the computed checksums. + * @param init Indicates if the SpookyHash checksum is initialized. + */ void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init); - void commit_update_checksums_from_tables(); - void CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables() - + /** + * @brief Performs the following actions: + * 1. Gets the current contents of table 'myhgm.TableName', using 'ColumnName' ordering. + * 2. Computes the checksum for that resultset. + * 3. Updates the supplied 'raw_checksum' and the supplied 'SpookyHash' with it. + * @details Stands for 'commit_update_checksum_from_table_1'. + * @param myhash Hash to be updated with the resultset checksum from the selected table. + * @param init If the supplied 'SpookyHash' has already being initialized. + * @param TableName The tablename from which to obtain the resultset for the 'raw_checksum' computation. + * @param ColumnName A column name to use for ordering in the supplied 'TableName'. + * @param raw_checksum A 'raw_checksum' to be updated with the obtained resultset. + */ + void CUCFT1( + SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum + ); /** * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime. * The store configuration is later used by Cluster to propagate current config. diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 559bcb2f40..5a12e62559 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -148,22 +148,38 @@ struct incoming_servers_t { // Separate structs for runtime mysql server and mysql server v2 to avoid human error struct runtime_mysql_servers_checksum_t { - std::string checksum; + std::string value; time_t epoch; runtime_mysql_servers_checksum_t(); - runtime_mysql_servers_checksum_t(const std::string& checksum, time_t epoch); + runtime_mysql_servers_checksum_t(const std::string& value, time_t epoch); }; struct mysql_servers_v2_checksum_t { - std::string checksum; + std::string value; time_t epoch; mysql_servers_v2_checksum_t(); - mysql_servers_v2_checksum_t(const std::string& checksum, time_t epoch); + mysql_servers_v2_checksum_t(const std::string& value, time_t epoch); }; // +struct peer_runtime_mysql_servers_t { + SQLite3_result* resultset { nullptr }; + runtime_mysql_servers_checksum_t checksum {}; + + peer_runtime_mysql_servers_t(); + peer_runtime_mysql_servers_t(SQLite3_result*, const runtime_mysql_servers_checksum_t&); +}; + +struct peer_mysql_servers_v2_t { + SQLite3_result* resultset { nullptr }; + mysql_servers_v2_checksum_t checksum {}; + + peer_mysql_servers_v2_t(); + peer_mysql_servers_v2_t(SQLite3_result*, const mysql_servers_v2_checksum_t&); +}; + class ProxySQL_Admin { private: volatile int main_shutdown; diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index acf52664d2..9869cbe09c 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -249,14 +249,5 @@ inline void replace_checksum_zeros(char* checksum) { */ std::string get_checksum_from_hash(uint64_t hash); -/** - * @brief Remove the rows from the resultset matching the supplied predicate. - * @param resultset The resultset which rows are to be removed. - * @param pred Predicate that should return 'true' for the rows to be removed. - */ -void remove_sqlite3_resultset_rows( - std::unique_ptr& resultset, const std::function& pred -); - void close_all_non_term_fd(std::vector excludeFDs); #endif diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 76e3893c84..34f4f72915 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1590,7 +1590,9 @@ SQLite3_result * MySQL_HostGroups_Manager::execute_query(char *query, char **err return resultset; } -void MySQL_HostGroups_Manager::CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum) { +void MySQL_HostGroups_Manager::CUCFT1( + SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum +) { char *error=NULL; int cols=0; int affected_rows=0; @@ -1599,8 +1601,14 @@ void MySQL_HostGroups_Manager::CUCFT1(const string& TableName, const string& Col mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); if (resultset) { if (resultset->rows_count) { - raw_checksum = resultset->raw_checksum(); - proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), raw_checksum); + if (init == false) { + init = true; + myhash.Init(19,3); + } + uint64_t hash1_ = resultset->raw_checksum(); + raw_checksum = hash1_; + myhash.Update(&hash1_, sizeof(hash1_)); + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), hash1_); } delete resultset; } else { @@ -1608,12 +1616,19 @@ void MySQL_HostGroups_Manager::CUCFT1(const string& TableName, const string& Col } } -void MySQL_HostGroups_Manager::commit_update_checksums_from_tables() { - CUCFT1("mysql_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); - CUCFT1("mysql_group_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GROUP_REPLICATION_HOSTGROUPS]); - CUCFT1("mysql_galera_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GALERA_HOSTGROUPS]); - CUCFT1("mysql_aws_aurora_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_AURORA_HOSTGROUPS]); - CUCFT1("mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); +void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& myhash, bool& init) { + // Always reset the current table values before recomputing + for (size_t i = 0; i < table_resultset_checksum.size(); i++) { + if (i != HGM_TABLES::MYSQL_SERVERS && i != HGM_TABLES::MYSQL_SERVERS_V2) { + table_resultset_checksum[i] = 0; + } + } + + CUCFT1(myhash,init,"mysql_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_group_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GROUP_REPLICATION_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_galera_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GALERA_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_aws_aurora_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_AURORA_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); } /** @@ -1689,29 +1704,22 @@ void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { } } -const char MYSQL_SERVERS_CHECKSUM_QUERY[] { - "SELECT hostgroup_id, hostname, port, gtid_port, CASE WHEN status=0 OR status=1 OR status=4 THEN 0 ELSE status END status," - " weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers" - " WHERE status<>3 ORDER BY hostgroup_id, hostname, port" -}; - /** - * @brief Generates a resultset which is used to compute the current 'mysql_servers' checksum. - * @details The resultset should report all servers status as ONLINE(0), with the exception of 'OFFLINE_HARD'. - * Servers with this status should be excluded from the resultset. + * @brief Generates a resultset holding the current Admin 'runtime_mysql_servers' as reported by Admin. + * @details Requires caller to hold the mutex 'MySQL_HostGroups_Manager::wrlock'. * @param mydb The db in which to perform the query, typically 'MySQL_HostGroups_Manager::mydb'. - * @return An SQLite3 resultset for the query 'MYSQL_SERVERS_CHECKSUM_QUERY'. + * @return An SQLite3 resultset for the query 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. */ -unique_ptr get_mysql_servers_checksum_resultset(SQLite3DB* mydb) { +unique_ptr get_admin_runtime_mysql_servers(SQLite3DB* mydb) { char* error = nullptr; int cols = 0; int affected_rows = 0; SQLite3_result* resultset = nullptr; - mydb->execute_statement(MYSQL_SERVERS_CHECKSUM_QUERY, &error, &cols, &affected_rows, &resultset); + mydb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); if (error) { - proxy_error("Checksum generation query for 'mysql_servers' failed with error '%s'\n", error); + proxy_error("SQLite3 query generating 'runtime_mysql_servers' resultset failed with error '%s'\n", error); assert(0); } @@ -1719,138 +1727,156 @@ unique_ptr get_mysql_servers_checksum_resultset(SQLite3DB* mydb) } /** - * @brief Generates a resultset holding the current Admin 'runtime_mysql_servers' as reported by Admin. - * @param mydb The db in which to perform the query, typically 'MySQL_HostGroups_Manager::mydb'. - * @return An SQLite3 resultset for the query 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. + * @brief Generates a resultset with holding the current 'mysql_servers_v2' table. + * @details Requires caller to hold the mutex 'ProxySQL_Admin::mysql_servers_wrlock'. + * @return A resulset holding 'mysql_servers_v2'. */ -unique_ptr get_admin_runtime_mysql_servers(SQLite3DB* mydb) { +unique_ptr get_mysql_servers_v2() { char* error = nullptr; int cols = 0; int affected_rows = 0; SQLite3_result* resultset = nullptr; - mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + if (GloAdmin && GloAdmin->admindb) { + GloAdmin->admindb->execute_statement( + MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset + ); + } return unique_ptr(resultset); } -/** - * @brief Removes rows with 'OFFLINE_HARD' servers in the supplied resultset. - * @details It assumes that the supplied resultset is generated via 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. - * @param resultset The resultset from which rows are to be removed. - */ -void remove_resultset_offline_hard_servers(unique_ptr& resultset) { - if (resultset->columns < 5) { - return; - } +void update_glovars_checksum_with_peers( + ProxySQL_Checksum_Value& module_checksum, + const string& new_checksum, + const string& peer_checksum_value, + time_t new_epoch, + time_t peer_checksum_epoch, + bool update_version +) { + module_checksum.set_checksum(const_cast(new_checksum.c_str())); - const auto is_offline = [] (SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - return true; - } else { - return false; - } - }; + if (update_version) + module_checksum.version++; - remove_sqlite3_resultset_rows(resultset, is_offline); + bool computed_checksum_matches = + peer_checksum_value != "" && module_checksum.checksum == peer_checksum_value; + + if (peer_checksum_epoch != 0 && computed_checksum_matches) { + module_checksum.epoch = peer_checksum_epoch; + } else { + module_checksum.epoch = new_epoch; + } } /** - * @brief Updates the global 'mysql_servers' checksum. + * @brief Updates the global 'mysql_servers' module checksum. * @details If the new computed checksum matches the supplied 'cluster_checksum', the epoch used for the - * checksum, is the supplied epoch instead of current time. This way we ensure the preservation of the + * checksum is the supplied epoch instead of current time. This way we ensure the preservation of the * checksum and epoch fetched from the ProxySQL cluster peer node. * - * @param new_checksum The new computed checksum by ProxySQL. - * @param old_checksum A checksum, previously fetched from ProxySQL cluster. Should be left empty if the - * update isn't considering this scenario. - * @param epoch The epoch to be preserved in case the supplied 'cluster_checksum' matches the new computed + * IMPORTANT: This function also generates a new 'global_checksum'. This is because everytime + * 'runtime_mysql_servers' change, updating the global checksum is unconditional. + * @param new_checksum The new computed checksum for 'runtime_mysql_servers'. + * @param peer_checksum A checksum fetched from another ProxySQL cluster node, holds the checksum value + * and its epoch. Should be empty if no remote checksum is being considered. + * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed * checksum. */ void update_glovars_mysql_servers_checksum( - const std::string& new_checksum, const std::string& cluster_checksum = "", const time_t epoch = 0 + const string& new_checksum, + const runtime_mysql_servers_checksum_t& peer_checksum = {}, + bool update_version = false ) { - GloVars.checksums_values.mysql_servers.set_checksum(const_cast(new_checksum.c_str())); - GloVars.checksums_values.mysql_servers.version++; - - time_t t = time(NULL); - bool computed_checksum_matches { - cluster_checksum != "" && GloVars.checksums_values.mysql_servers.checksum == cluster_checksum - }; - - if (epoch != 0 && computed_checksum_matches) { - GloVars.checksums_values.mysql_servers.epoch = epoch; - } else { - GloVars.checksums_values.mysql_servers.epoch = t; - } + time_t new_epoch = time(NULL); + + update_glovars_checksum_with_peers( + GloVars.checksums_values.mysql_servers, + new_checksum, + peer_checksum.value, + new_epoch, + peer_checksum.epoch, + update_version + ); GloVars.checksums_values.updates_cnt++; GloVars.generate_global_checksum(); - GloVars.epoch_version = t; + GloVars.epoch_version = new_epoch; +} + +/** + * @brief Updates the global 'mysql_servers_v2' module checksum. + * @details Unlike 'update_glovars_mysql_servers_checksum' this function doesn't generate a new + * 'global_checksum'. It's caller responsibility to ensure that 'global_checksum' is updated. + * @param new_checksum The new computed checksum for 'mysql_servers_v2'. + * @param peer_checksum A checksum fetched from another ProxySQL cluster node, holds the checksum value + * and its epoch. Should be empty if no remote checksum is being considered. + * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed + * checksum. + */ +void update_glovars_mysql_servers_v2_checksum( + const string& new_checksum, + const mysql_servers_v2_checksum_t& peer_checksum = {}, + bool update_version = false +) { + time_t new_epoch = time(NULL); + + update_glovars_checksum_with_peers( + GloVars.checksums_values.mysql_servers_v2, + new_checksum, + peer_checksum.value, + new_epoch, + peer_checksum.epoch, + update_version + ); } -void MySQL_HostGroups_Manager::commit_generate_mysql_servers_table(SQLite3_result* runtime_mysql_servers) { +uint64_t MySQL_HostGroups_Manager::commit_update_checksum_from_mysql_servers(SQLite3_result* runtime_mysql_servers) { mydb->execute("DELETE FROM mysql_servers"); generate_mysql_servers_table(); if (runtime_mysql_servers == nullptr) { unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; - - // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster nodes, or - // relevant for checksum computation. If this step isn't performed, this could cause mismatching - // checksums between different primary nodes in a ProxySQL cluster, since OFFLINE_HARD servers - // preservation depends on unknown and unpredictable connections conditions. - remove_resultset_offline_hard_servers(resultset); save_runtime_mysql_servers(resultset.release()); } else { save_runtime_mysql_servers(runtime_mysql_servers); } -} - -void MySQL_HostGroups_Manager::commit_update_checksum_from_mysql_servers(SpookyHash& myhash, bool& init) { - unique_ptr mysrvs_checksum_resultset { get_mysql_servers_checksum_resultset(mydb) }; - // Reset table checksum value before recomputing - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = 0; + uint64_t raw_checksum = this->runtime_mysql_servers ? this->runtime_mysql_servers->raw_checksum() : 0; + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = raw_checksum; - if (mysrvs_checksum_resultset) { - if (mysrvs_checksum_resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = mysrvs_checksum_resultset->raw_checksum(); - table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = hash1_; + return raw_checksum; +} - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); - } +uint64_t MySQL_HostGroups_Manager::commit_update_checksum_from_mysql_servers_v2(SQLite3_result* mysql_servers_v2) { + if (mysql_servers_v2 == nullptr) { + unique_ptr resultset { get_mysql_servers_v2() }; + save_mysql_servers_v2(resultset.release()); } else { - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); + save_mysql_servers_v2(mysql_servers_v2); } -} -std::string MySQL_HostGroups_Manager::gen_global_mysql_servers_checksum() { - SpookyHash global_hash; - bool init = false; + uint64_t raw_checksum = this->incoming_mysql_servers_v2 ? this->incoming_mysql_servers_v2->raw_checksum() : 0; + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_V2] = raw_checksum; - // Regenerate 'mysql_servers' and generate new checksum, initialize the new 'global_hash' - commit_update_checksum_from_mysql_servers(global_hash, init); - - // Complete the hash with the rest of the unchanged modules - for (size_t i = 0; i < table_resultset_checksum.size(); i++) { - uint64_t hash_val = table_resultset_checksum[i]; + return raw_checksum; +} - if (i != HGM_TABLES::MYSQL_SERVERS && hash_val != 0) { - if (init == false) { - init = true; - global_hash.Init(19, 3); - } +std::string MySQL_HostGroups_Manager::gen_global_mysql_servers_v2_checksum(uint64_t servers_v2_hash) { + bool init = false; + SpookyHash global_hash {}; - global_hash.Update(&hash_val, sizeof(hash_val)); + if (servers_v2_hash != 0) { + if (init == false) { + init = true; + global_hash.Init(19, 3); } + + global_hash.Update(&servers_v2_hash, sizeof(servers_v2_hash)); } + commit_update_checksums_from_tables(global_hash, init); + uint64_t hash_1 = 0, hash_2 = 0; if (init) { global_hash.Final(&hash_1,&hash_2); @@ -1861,9 +1887,10 @@ std::string MySQL_HostGroups_Manager::gen_global_mysql_servers_checksum() { } bool MySQL_HostGroups_Manager::commit( - SQLite3_result* runtime_mysql_servers, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server, - SQLite3_result* mysql_servers_v2, const mysql_servers_v2_checksum_t& peer_mysql_server_v2, - bool only_commit_runtime_mysql_servers, bool update_version + const peer_runtime_mysql_servers_t& peer_runtime_mysql_servers, + const peer_mysql_servers_v2_t& peer_mysql_servers_v2, + bool only_commit_runtime_mysql_servers, + bool update_version ) { // if only_commit_runtime_mysql_servers is true, mysql_servers_v2 resultset will not be entertained and will cause memory leak. if (only_commit_runtime_mysql_servers) { @@ -2077,9 +2104,10 @@ bool MySQL_HostGroups_Manager::commit( has_gtid_port = false; } if (resultset) { delete resultset; resultset=NULL; } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); + mydb->execute("DELETE FROM mysql_servers_incoming"); - uint64_t mysql_servers_v2_checksum = 0; - + string global_checksum_v2 {}; if (only_commit_runtime_mysql_servers == false) { // replication if (incoming_replication_hostgroups) { // this IF is extremely important, otherwise replication hostgroups may disappear @@ -2116,58 +2144,31 @@ bool MySQL_HostGroups_Manager::commit( generate_mysql_hostgroup_attributes_table(); } - mysql_servers_v2_checksum = get_mysql_servers_v2_checksum(mysql_servers_v2); - } + uint64_t new_hash = commit_update_checksum_from_mysql_servers_v2(peer_mysql_servers_v2.resultset); - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); - mydb->execute("DELETE FROM mysql_servers_incoming"); - - // regenerating mysql_servers records - mydb->execute("DELETE FROM mysql_servers"); - generate_mysql_servers_table(); - - const auto mysql_servers_checksum = get_mysql_servers_checksum(runtime_mysql_servers); - - char buf[80]; - uint32_t d32[2]; - const time_t t = time(NULL); - - memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); - sprintf(buf, "0x%0X%0X", d32[0], d32[1]); - pthread_mutex_lock(&GloVars.checksum_mutex); - GloVars.checksums_values.mysql_servers.set_checksum(buf); - - if (update_version) - GloVars.checksums_values.mysql_servers.version++; + { + const string new_checksum { get_checksum_from_hash(new_hash) }; + proxy_info("Checksum for table %s is %s\n", "mysql_servers_v2", new_checksum.c_str()); + } - if (peer_runtime_mysql_server.epoch != 0 && peer_runtime_mysql_server.checksum.empty() == false && - GloVars.checksums_values.mysql_servers.checksum == peer_runtime_mysql_server.checksum) { - GloVars.checksums_values.mysql_servers.epoch = peer_runtime_mysql_server.epoch; - } else { - GloVars.checksums_values.mysql_servers.epoch = t; + global_checksum_v2 = gen_global_mysql_servers_v2_checksum(new_hash); + proxy_info("New computed global checksum for 'mysql_servers_v2' is '%s'\n", global_checksum_v2.c_str()); } - - if (only_commit_runtime_mysql_servers == false) { - memcpy(&d32, &mysql_servers_v2_checksum, sizeof(mysql_servers_v2_checksum)); - sprintf(buf, "0x%0X%0X", d32[0], d32[1]); - GloVars.checksums_values.mysql_servers_v2.set_checksum(buf); - if (update_version) - GloVars.checksums_values.mysql_servers_v2.version++; + // Update 'mysql_servers' and global checksums + { + uint64_t new_hash = commit_update_checksum_from_mysql_servers(peer_runtime_mysql_servers.resultset); + const string new_checksum { get_checksum_from_hash(new_hash) }; + proxy_info("Checksum for table %s is %s\n", "mysql_servers", new_checksum.c_str()); - if (peer_mysql_server_v2.epoch != 0 && peer_mysql_server_v2.checksum.empty() == false && - GloVars.checksums_values.mysql_servers_v2.checksum == peer_mysql_server_v2.checksum) { - GloVars.checksums_values.mysql_servers_v2.epoch = peer_mysql_server_v2.epoch; - } else { - GloVars.checksums_values.mysql_servers_v2.epoch = t; + pthread_mutex_lock(&GloVars.checksum_mutex); + if (only_commit_runtime_mysql_servers == false) { + update_glovars_mysql_servers_v2_checksum(global_checksum_v2, peer_mysql_servers_v2.checksum, true); } + update_glovars_mysql_servers_checksum(new_checksum, peer_runtime_mysql_servers.checksum, update_version); + pthread_mutex_unlock(&GloVars.checksum_mutex); } - GloVars.checksums_values.updates_cnt++; - GloVars.generate_global_checksum(); - GloVars.epoch_version = t; - pthread_mutex_unlock(&GloVars.checksum_mutex); - // fill Hostgroup_Manager_Mapping with latest records update_hostgroup_manager_mappings(); @@ -2241,121 +2242,6 @@ uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* ru return table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; } -/** - * @brief Computes checksum for the admin mysql_server resultset and generates an accumulated checksum by including the following modules: - * MYSQL_REPLICATION_HOSTGROUPS, MYSQL_GROUP_REPLICATION_HOSTGROUPS, MYSQL_GALERA_HOSTGROUPS, and MYSQL_HOSTGROUP_ATTRIBUTES. - * - * @details The 'incoming_mysql_servers_v2' includes the same records as 'main.mysql_servers'. If this set of records is empty, then the - * records are retrieved from 'main.mysql_servers'. - * - * @param incoming_mysql_servers_v2 resultset of admin mysql_servers or can be a nullptr. - */ -uint64_t MySQL_HostGroups_Manager::get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2) { - - //Note: GloVars.checksum_mutex needs to be locked - SQLite3_result* resultset = nullptr; - - if (incoming_mysql_servers_v2 == nullptr) - { - char* error = nullptr; - int cols = 0; - int affected_rows = 0; - - GloAdmin->admindb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset); - - if (resultset) { - save_mysql_servers_v2(resultset); - } else { - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_v2", (long unsigned int)0); - } - } else { - resultset = incoming_mysql_servers_v2; - save_mysql_servers_v2(incoming_mysql_servers_v2); - } - - // reset checksum excluding MYSQL_SERVERS_V2 - table_resultset_checksum[MYSQL_REPLICATION_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_GROUP_REPLICATION_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_GALERA_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_AWS_AURORA_HOSTGROUPS] = 0; - table_resultset_checksum[MYSQL_HOSTGROUP_ATTRIBUTES] = 0; - - table_resultset_checksum[MYSQL_SERVERS_V2] = resultset != nullptr ? resultset->raw_checksum() : 0; - proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers_v2", table_resultset_checksum[MYSQL_SERVERS_V2]); - - commit_update_checksums_from_tables(); - - uint64_t hash1 = 0, hash2 = 0; - SpookyHash myhash; - bool init = false; - - hash1 = table_resultset_checksum[MYSQL_SERVERS_V2]; - if (hash1) { - if (init == false) { - init = true; - myhash.Init(19, 3); - } - - myhash.Update(&hash1, sizeof(hash1)); - } - - hash1 = table_resultset_checksum[MYSQL_REPLICATION_HOSTGROUPS]; - if (hash1) { - if (init == false) { - init = true; - myhash.Init(19, 3); - } - - myhash.Update(&hash1, sizeof(hash1)); - } - - hash1 = table_resultset_checksum[MYSQL_GROUP_REPLICATION_HOSTGROUPS]; - if (hash1) { - if (init == false) { - init = true; - myhash.Init(19, 3); - } - - myhash.Update(&hash1, sizeof(hash1)); - } - - hash1 = table_resultset_checksum[MYSQL_GALERA_HOSTGROUPS]; - if (hash1) { - if (init == false) { - init = true; - myhash.Init(19, 3); - } - - myhash.Update(&hash1, sizeof(hash1)); - } - - hash1 = table_resultset_checksum[MYSQL_AWS_AURORA_HOSTGROUPS]; - if (hash1) { - if (init == false) { - init = true; - myhash.Init(19, 3); - } - - myhash.Update(&hash1, sizeof(hash1)); - } - - hash1 = table_resultset_checksum[MYSQL_HOSTGROUP_ATTRIBUTES]; - if (hash1) { - if (init == false) { - init = true; - myhash.Init(19, 3); - } - - myhash.Update(&hash1, sizeof(hash1)); - } - - if (init == true) { - myhash.Final(&hash1, &hash2); - } - - return hash1; -} - bool MySQL_HostGroups_Manager::gtid_exists(MySrvC *mysrvc, char * gtid_uuid, uint64_t gtid_trxid) { bool ret = false; pthread_rwlock_rdlock(>id_rwlock); @@ -5070,23 +4956,23 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listexecute("DELETE FROM mysql_servers"); generate_mysql_servers_table(); - const auto mysql_servers_checksum = get_mysql_servers_checksum(); - hgsm_mysql_servers_checksum = mysql_servers_checksum; + // Update the global checksums after 'mysql_servers' regeneration + { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + uint64_t raw_checksum = resultset ? resultset->raw_checksum() : 0; - char buf[ProxySQL_Checksum_Value_LENGTH]; - uint32_t d32[2]; - memcpy(&d32, &mysql_servers_checksum, sizeof(mysql_servers_checksum)); - sprintf(buf, "0x%0X%0X", d32[0], d32[1]); - pthread_mutex_lock(&GloVars.checksum_mutex); - GloVars.checksums_values.mysql_servers.set_checksum(buf); - time_t t = time(NULL); + // This is required to be updated to avoid extra rebuilding member 'hostgroup_server_mapping' + // during 'commit'. For extra details see 'hgsm_mysql_servers_checksum' @details. + hgsm_mysql_servers_checksum = raw_checksum; - GloVars.checksums_values.mysql_servers.epoch = t; + string mysrvs_checksum { get_checksum_from_hash(raw_checksum) }; + save_runtime_mysql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); - GloVars.checksums_values.updates_cnt++; - GloVars.generate_global_checksum(); - GloVars.epoch_version = t; - pthread_mutex_unlock(&GloVars.checksum_mutex); + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } } wrunlock(); unsigned long long curtime2 = monotonic_time(); @@ -5980,11 +5866,10 @@ void MySQL_HostGroups_Manager::update_group_replication_add_autodiscovered( // Update the global checksums after 'mysql_servers' regeneration { unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; - remove_resultset_offline_hard_servers(resultset); + string mysrvs_checksum { get_checksum_from_hash(resultset ? resultset->raw_checksum() : 0) }; save_runtime_mysql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); - string mysrvs_checksum { gen_global_mysql_servers_checksum() }; - proxy_info("New computed global checksum for 'mysql_servers' is '%s'\n", mysrvs_checksum.c_str()); pthread_mutex_lock(&GloVars.checksum_mutex); update_glovars_mysql_servers_checksum(mysrvs_checksum); pthread_mutex_unlock(&GloVars.checksum_mutex); @@ -7955,10 +7840,10 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid // Update the global checksums after 'mysql_servers' regeneration { unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; - remove_resultset_offline_hard_servers(resultset); + string mysrvs_checksum { get_checksum_from_hash(resultset ? resultset->raw_checksum() : 0) }; save_runtime_mysql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); - string mysrvs_checksum { gen_global_mysql_servers_checksum() }; pthread_mutex_lock(&GloVars.checksum_mutex); update_glovars_mysql_servers_checksum(mysrvs_checksum); pthread_mutex_unlock(&GloVars.checksum_mutex); @@ -8105,11 +7990,10 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid // Update the global checksums after 'mysql_servers' regeneration { unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; - remove_resultset_offline_hard_servers(resultset); + string mysrvs_checksum { get_checksum_from_hash(resultset ? resultset->raw_checksum() : 0) }; save_runtime_mysql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); - string mysrvs_checksum { gen_global_mysql_servers_checksum() }; - proxy_info("New computed global checksum for 'mysql_servers' is '%s'\n", mysrvs_checksum.c_str()); pthread_mutex_lock(&GloVars.checksum_mutex); update_glovars_mysql_servers_checksum(mysrvs_checksum); pthread_mutex_unlock(&GloVars.checksum_mutex); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index cb3824caae..2b49d3c02e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1097,12 +1097,12 @@ incoming_servers_t::incoming_servers_t( runtime_mysql_servers_checksum_t::runtime_mysql_servers_checksum_t() : epoch(0) {} runtime_mysql_servers_checksum_t::runtime_mysql_servers_checksum_t(const std::string& checksum, time_t epoch) : - checksum(checksum), epoch(epoch) {} + value(checksum), epoch(epoch) {} mysql_servers_v2_checksum_t::mysql_servers_v2_checksum_t() : epoch(0) {} mysql_servers_v2_checksum_t::mysql_servers_v2_checksum_t(const std::string& checksum, time_t epoch) : - checksum(checksum), epoch(epoch) {} + value(checksum), epoch(epoch) {} bootstrap_info_t::~bootstrap_info_t() { if (servers != nullptr) { @@ -1113,6 +1113,20 @@ bootstrap_info_t::~bootstrap_info_t() { } } +peer_runtime_mysql_servers_t::peer_runtime_mysql_servers_t() : resultset(nullptr), checksum() {} + +peer_runtime_mysql_servers_t::peer_runtime_mysql_servers_t( + SQLite3_result* resultset, const runtime_mysql_servers_checksum_t& checksum +) : resultset(resultset), checksum(checksum) +{} + +peer_mysql_servers_v2_t::peer_mysql_servers_v2_t() : resultset(nullptr), checksum() {} + +peer_mysql_servers_v2_t::peer_mysql_servers_v2_t( + SQLite3_result* resultset, const mysql_servers_v2_checksum_t& checksum +) : resultset(resultset), checksum(checksum) +{} + int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { int r = 0; if (!GloQPro) return 0; @@ -13082,7 +13096,11 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc } // commit all the changes - MyHGM->commit(runtime_mysql_servers, peer_runtime_mysql_server, incoming_mysql_servers_v2, peer_mysql_server_v2, false, true); + MyHGM->commit( + { runtime_mysql_servers, peer_runtime_mysql_server }, + { incoming_mysql_servers_v2, peer_mysql_server_v2 }, + false, true + ); // quering runtime table will update and return latest records, so this is not needed. // GloAdmin->save_mysql_servers_runtime_to_database(true); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 0b5a650cc8..fdd7e5a3e7 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1805,7 +1805,10 @@ void ProxySQL_Cluster::pull_runtime_mysql_servers_from_peer(const runtime_mysql_ proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading runtime_mysql_servers from peer %s:%d into mysql_servers_incoming", hostname, port); MyHGM->servers_add(runtime_mysql_servers_resultset.get()); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Updating runtime_mysql_servers from peer %s:%d", hostname, port); - MyHGM->commit(runtime_mysql_servers_resultset.release(), peer_runtime_mysql_server, nullptr, {}, true, true); + MyHGM->commit( + { runtime_mysql_servers_resultset.release(), peer_runtime_mysql_server }, + { nullptr, {} }, true, true + ); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving Runtime MySQL Servers to Database\n"); diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index 21bbe7ccb4..4e74b2df8c 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -455,14 +455,6 @@ std::string get_checksum_from_hash(uint64_t hash) { return string { &s_buf.front() }; } -void remove_sqlite3_resultset_rows( - unique_ptr& resultset, const function& pred -) { - const auto remove_it { std::remove_if(resultset->rows.begin(), resultset->rows.end(), pred) }; - resultset->rows.erase(remove_it, resultset->rows.end()); - resultset->rows_count = resultset->rows.size(); -} - void close_all_non_term_fd(std::vector excludeFDs) { DIR *d; struct dirent *dir; From 055176b4d079c4e3010e4f91a20ac435c348cb9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 19 Jul 2023 14:49:19 +0200 Subject: [PATCH 53/56] Fix minor compiler warning in 'proxy_debug' call --- lib/ProxySQL_Cluster.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index fdd7e5a3e7..e7b975bd8d 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -2963,7 +2963,7 @@ bool ProxySQL_Cluster_Nodes::Update_Global_Checksum(char * _h, uint16_t _p, MYSQ proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum 0x%llX for peer %s:%d matches\n", v, node->get_hostname(), node->get_port()); ret = false; } else { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum for peer %s:%d is different from fetched one. Local checksum:[0x%lX] Fetched checksum:[0x%lX]\n", node->get_hostname(), node->get_port(), node->global_checksum, v); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum for peer %s:%d is different from fetched one. Local checksum:[0x%lX] Fetched checksum:[0x%llX]\n", node->get_hostname(), node->get_port(), node->global_checksum, v); node->global_checksum = v; } } From f5f56ef7ccd86f75ce03209f130610e5bd8cb764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 19 Jul 2023 14:50:43 +0200 Subject: [PATCH 54/56] Fix crash during shutdown due to 'mysql_servers_v2' global checksum --- lib/ProxySQL_GloVars.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index 778febaef2..238624ea27 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -89,6 +89,7 @@ ProxySQL_GlobalVariables::~ProxySQL_GlobalVariables() { checksums_values.admin_variables.in_shutdown = true; checksums_values.mysql_query_rules.in_shutdown = true; checksums_values.mysql_servers.in_shutdown = true; + checksums_values.mysql_servers_v2.in_shutdown = true; checksums_values.mysql_users.in_shutdown = true; checksums_values.mysql_variables.in_shutdown = true; checksums_values.ldap_variables.in_shutdown = true; From c3e932a6416e280002677878147d3c6e299acf25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 19 Jul 2023 17:15:20 +0200 Subject: [PATCH 55/56] Remove outdated commented code from Aurora implementation --- lib/MySQL_HostGroups_Manager.cpp | 140 +++++-------------------------- 1 file changed, 21 insertions(+), 119 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 34f4f72915..556c1351ea 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7610,26 +7610,8 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid free(error); error=NULL; } - //free(query); if (resultset) { -/* - // let's get info about this cluster - pthread_mutex_lock(&AWS_Aurora_Info_mutex); - std::map::iterator it2; - it2 = AWS_Aurora_Info_Map.find(_writer_hostgroup); - AWS_Aurora_Info *info=NULL; - if (it2!=AWS_Aurora_Info_Map.end()) { - info=it2->second; - writer_is_also_reader=info->writer_is_also_reader; - new_reader_weight = info->new_reader_weight; - read_HG = info->reader_hostgroup; - //need_converge=info->need_converge; - //info->need_converge=false; - //max_writers = info->max_writers; - } - pthread_mutex_unlock(&AWS_Aurora_Info_mutex); -*/ if (resultset->rows_count) { for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; @@ -7644,41 +7626,17 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid } } } -/* - if (need_converge == false) { - SQLite3_result *resultset2=NULL; - q = (char *)"SELECT COUNT(*) FROM mysql_servers WHERE hostgroup_id=%d AND status=0"; - query=(char *)malloc(strlen(q)+32); - sprintf(query,q,_writer_hostgroup); - mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset2); - if (resultset2) { - if (resultset2->rows_count) { - for (std::vector::iterator it = resultset2->rows.begin() ; it != resultset2->rows.end(); ++it) { - SQLite3_row *r=*it; - int nwriters = atoi(r->fields[0]); - if (nwriters > max_writers) { - proxy_warning("Galera: too many writers in HG %d. Max=%d, current=%d\n", _writer_hostgroup, max_writers, nwriters); - need_converge = true; - } - } - } - delete resultset2; + + if (found_writer) { // maybe no-op + if ( + (writer_is_also_reader==0 && found_reader==false) + || + (writer_is_also_reader > 0 && found_reader==true) + ) { // either both true or both false + delete resultset; + resultset=NULL; } - free(query); } -*/ -// if (need_converge==false) { - if (found_writer) { // maybe no-op - if ( - (writer_is_also_reader==0 && found_reader==false) - || - (writer_is_also_reader > 0 && found_reader==true) - ) { // either both true or both false - delete resultset; - resultset=NULL; - } - } -// } } if (resultset) { @@ -7686,8 +7644,6 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid // This should be the case most of the time, // because the calling function knows if an action is required. if (resultset->rows_count) { - //need_converge=false; - GloAdmin->mysql_servers_wrlock(); mydb->execute("DELETE FROM mysql_servers_incoming"); q=(char *)"INSERT INTO mysql_servers_incoming SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE hostgroup_id=%d"; @@ -7697,17 +7653,12 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid sprintf(query, q, _writer_hostgroup, _server_id, domain_name, aurora_port); mydb->execute(query); q=(char *)"UPDATE OR IGNORE mysql_servers_incoming SET hostgroup_id=%d WHERE hostname='%s%s' AND port=%d AND hostgroup_id<>%d"; - //query=(char *)malloc(strlen(q)+strlen(_hostname)+1024); // increased this buffer as it is used for other queries too sprintf(query, q, _writer_hostgroup, _server_id, domain_name, aurora_port, _writer_hostgroup); mydb->execute(query); - //free(query); q=(char *)"DELETE FROM mysql_servers_incoming WHERE hostname='%s%s' AND port=%d AND hostgroup_id<>%d"; - //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); sprintf(query, q, _server_id, domain_name, aurora_port, _writer_hostgroup); mydb->execute(query); - //free(query); q=(char *)"UPDATE mysql_servers_incoming SET status=0 WHERE hostname='%s%s' AND port=%d AND hostgroup_id=%d"; - //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); sprintf(query, q, _server_id, domain_name, aurora_port, _writer_hostgroup); mydb->execute(query); @@ -7719,7 +7670,6 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid sprintf(query,q,_rhid, new_reader_weight, _whid); mydb->execute(query); - //free(query); if (writer_is_also_reader && read_HG>=0) { q=(char *)"INSERT OR IGNORE INTO mysql_servers_incoming (hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment) SELECT %d,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_incoming WHERE hostgroup_id=%d AND hostname='%s%s' AND port=%d"; sprintf(query, q, read_HG, _writer_hostgroup, _server_id, domain_name, aurora_port); @@ -7772,30 +7722,11 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid mydb->execute(query); commit(); wrlock(); -/* - SQLite3_result *resultset2=NULL; - q=(char *)"SELECT writer_hostgroup, reader_hostgroup FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=%d"; - sprintf(query,q,_writer_hostgroup); - mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset2); - if (resultset2) { - if (resultset2->rows_count) { - for (std::vector::iterator it = resultset2->rows.begin() ; it != resultset2->rows.end(); ++it) { - SQLite3_row *r=*it; - int writer_hostgroup=atoi(r->fields[0]); - int reader_hostgroup=atoi(r->fields[1]); -*/ - q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)"; - sprintf(query,q,_whid,_rhid); - mydb->execute(query); - generate_mysql_servers_table(&_whid); - generate_mysql_servers_table(&_rhid); -/* - } - } - delete resultset2; - resultset2=NULL; - } -*/ + q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)"; + sprintf(query,q,_whid,_rhid); + mydb->execute(query); + generate_mysql_servers_table(&_whid); + generate_mysql_servers_table(&_rhid); wrunlock(); } else { if (GloMTH->variables.hostgroup_manager_verbose > 1) { @@ -7916,52 +7847,23 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid query=(char *)malloc(strlen(q)+strlen(_server_id)+strlen(domain_name)+512); sprintf(query, q, _rhid, _server_id, domain_name, aurora_port, _whid); mydb->execute(query); - //free(query); // Reader could previously be also a reader, in which case previous operation 'UPDATE OR IGNORE' // did nothing. If server is still in the 'writer_hostgroup', we should remove it. q=(char *)"DELETE FROM mysql_servers_incoming WHERE hostname='%s%s' AND port=%d AND hostgroup_id=%d"; - //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); sprintf(query, q, _server_id, domain_name, aurora_port, _whid); mydb->execute(query); - //free(query); q=(char *)"UPDATE mysql_servers_incoming SET status=0 WHERE hostname='%s%s' AND port=%d AND hostgroup_id=%d"; sprintf(query, q, _server_id, domain_name, aurora_port, _rhid); mydb->execute(query); - //free(query); - //converge_galera_config(_writer_hostgroup); commit(); wrlock(); -/* - SQLite3_result *resultset2=NULL; - q=(char *)"SELECT writer_hostgroup, reader_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d"; - //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); - sprintf(query,q,_writer_hostgroup); - mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset2); - if (resultset2) { - if (resultset2->rows_count) { - for (std::vector::iterator it = resultset2->rows.begin() ; it != resultset2->rows.end(); ++it) { - SQLite3_row *r=*it; - int writer_hostgroup=atoi(r->fields[0]); - int backup_writer_hostgroup=atoi(r->fields[1]); - int reader_hostgroup=atoi(r->fields[2]); - int offline_hostgroup=atoi(r->fields[3]); -*/ - q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)"; - sprintf(query,q,_whid,_rhid); - mydb->execute(query); - generate_mysql_servers_table(&_whid); - generate_mysql_servers_table(&_rhid); -/* - generate_mysql_servers_table(&writer_hostgroup); - generate_mysql_servers_table(&backup_writer_hostgroup); - generate_mysql_servers_table(&reader_hostgroup); - generate_mysql_servers_table(&offline_hostgroup); - } - } - delete resultset2; - resultset2=NULL; - } -*/ + + q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)"; + sprintf(query,q,_whid,_rhid); + mydb->execute(query); + generate_mysql_servers_table(&_whid); + generate_mysql_servers_table(&_rhid); + wrunlock(); GloAdmin->mysql_servers_wrunlock(); free(query); From 9b00eb33a592ef191d97c0bb70236cb13127c2f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 11 Jan 2024 11:22:51 +0100 Subject: [PATCH 56/56] Make 'test_cluster_sync_mysql_servers' exit gracefully in clusterless env Current GitHub actions testing environment lacks of a ProxySQL cluster. This change allows the test to gracefully exit when the env conditions for it are not met. --- .../test_cluster_sync_mysql_servers-t.cpp | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp index 8d258710fe..a1338a4a8d 100644 --- a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -53,6 +53,8 @@ #include "command_line.h" #include "utils.h" +using std::string; + #define MYSQL_QUERY__(mysql, query) \ do { \ if (mysql_query(mysql, query)) { \ @@ -863,6 +865,28 @@ int main(int, char**) { std::vector core_nodes { extract_mysql_rows(my_res) }; mysql_free_result(my_res); + // 2.1 If core nodes are not reachable, assume no cluster is running; make test gracefully exit + if (core_nodes.size()) { + const string host { core_nodes[0][0] }; + const int port = std::stol(core_nodes[0][1]); + MYSQL* c_node_admin = mysql_init(NULL); + + if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { + int myerrno = mysql_errno(c_node_admin); + + if (myerrno == 2002) { + diag("Unable to connect to cluster Core nodes; required environment not met, gracefully exiting..."); + plan(0); + return exit_status(); + } else { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + return EXIT_FAILURE; + } + } + + mysql_close(c_node_admin); + } + // 3. Wait for all Core nodes to sync (confirm primary out of core nodes) std::string check_no_primary_query {}; string_format(