From 5fc840b9908143bfb5fc513fabbda07fdec98005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 11 Jan 2023 23:20:35 +0000 Subject: [PATCH 01/13] Removing commented code --- lib/MySQL_HostGroups_Manager.cpp | 149 ------------------------------- 1 file changed, 149 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index b32fa8852f..df88ca14f8 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -63,108 +63,6 @@ class MySrvC; class MySrvList; class MyHGC; -/* -class HGM_query_errors_stats { - public: - int hid; - char *hostname; - int port; - char *username; - char *schemaname; - int error_no; - unsigned int count_star; - time_t first_seen; - time_t last_seen; - char *last_error; - HGM_query_errors_stats(int _h, char *_hn, int _p, char *u, char *s, int e, char *le) { - hid=_h; - hostname=strdup(_hn); - port=_p; - username=strdup(u); - schemaname=strdup(s); - error_no=e; - last_error=strdup(le); - count_star=0; - first_seen=0; - last_seen=0; - } - void add_time(unsigned long long n, char *le) { - count_star++; - if (first_seen==0) { - first_seen=n; - } - last_seen=n; - if (strcmp(last_error,le)){ - free(last_error); - last_error=strdup(le); - } - } - ~HGM_query_errors_stats() { - if (hostname) { - free(hostname); - hostname=NULL; - } - if (username) { - free(username); - username=NULL; - } - if (schemaname) { - free(schemaname); - schemaname=NULL; - } - if (last_error) { - free(last_error); - last_error=NULL; - } - } - char **get_row() { - char buf[128]; - char **pta=(char **)malloc(sizeof(char *)*10); - sprintf(buf,"%d",hid); - pta[0]=strdup(buf); - assert(hostname); - pta[1]=strdup(hostname); - sprintf(buf,"%d",port); - pta[2]=strdup(buf); - assert(username); - pta[3]=strdup(username); - assert(schemaname); - pta[4]=strdup(schemaname); - sprintf(buf,"%d",error_no); - pta[5]=strdup(buf); - - sprintf(buf,"%u",count_star); - pta[6]=strdup(buf); - - time_t __now; - time(&__now); - unsigned long long curtime=monotonic_time(); - time_t seen_time; - - seen_time= __now - curtime/1000000 + first_seen/1000000; - sprintf(buf,"%ld", seen_time); - pta[7]=strdup(buf); - - seen_time= __now - curtime/1000000 + last_seen/1000000; - sprintf(buf,"%ld", seen_time); - pta[8]=strdup(buf); - - assert(last_error); - pta[9]=strdup(last_error); - return pta; - } - void free_row(char **pta) { - int i; - for (i=0;i<10;i++) { - assert(pta[i]); - free(pta[i]); - } - free(pta); - } -}; - -*/ - //static struct ev_async * gtid_ev_async; static pthread_mutex_t ev_loop_mutex; @@ -1910,53 +1808,6 @@ bool MySQL_HostGroups_Manager::commit( SpookyHash myhash; char buf[80]; bool init = false; -/* removing all this code, because we need them ordered - MySrvC *mysrvc=NULL; - for (unsigned int i=0; ilen; i++) { - MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); - for (unsigned int j=0; jmysrvs->servers->len; j++) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - mysrvc=myhgc->mysrvs->idx(j); - // hostgroup - sprintf(buf,"%u",mysrvc->myhgc->hid); - myhash.Update(buf,strlen(buf)); - // hoatname - if (mysrvc->address) { - myhash.Update(mysrvc->address,strlen(mysrvc->address)); - } else { myhash.Update("",0); } - // port - sprintf(buf,"%u",mysrvc->port); - myhash.Update(buf,strlen(buf)); - // status - sprintf(buf,"%u",mysrvc->status); - myhash.Update(buf,strlen(buf)); - // weight - sprintf(buf,"%u",mysrvc->weight); - myhash.Update(buf,strlen(buf)); - // compression - sprintf(buf,"%u",mysrvc->compression); - myhash.Update(buf,strlen(buf)); - // max_connections - sprintf(buf,"%u",mysrvc->max_connections); - myhash.Update(buf,strlen(buf)); - // max_replication_lag - sprintf(buf,"%u",mysrvc->max_replication_lag); - myhash.Update(buf,strlen(buf)); - // use_ssl - sprintf(buf,"%u",mysrvc->use_ssl); - myhash.Update(buf,strlen(buf)); - // max_latency_ms - sprintf(buf,"%u",mysrvc->max_latency_us); - myhash.Update(buf,strlen(buf)); - if (mysrvc->comment) { - myhash.Update(mysrvc->comment,strlen(mysrvc->comment)); - } else { myhash.Update("",0); } - } - } -*/ { mydb->execute("DELETE FROM mysql_servers"); generate_mysql_servers_table(); From 71e2510ef63488a0219fdd86f97f0945aae7ab2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 12 Jan 2023 01:35:08 +0000 Subject: [PATCH 02/13] Code cleanup on MySQL_HostGroups_Manager Some rework to try to reduce duplicated code --- include/MySQL_HostGroups_Manager.h | 25 ++- lib/ClickHouse_Authentication.cpp | 3 + lib/ClickHouse_Server.cpp | 3 + lib/MySQL_Authentication.cpp | 3 + lib/MySQL_HostGroups_Manager.cpp | 276 +++++++++-------------------- lib/MySQL_PreparedStatement.cpp | 3 + lib/MySQL_Session.cpp | 1 - lib/MySQL_Thread.cpp | 1 - lib/MySQL_Variables.cpp | 3 + lib/ProxySQL_Admin.cpp | 125 +++---------- lib/ProxySQL_Cluster.cpp | 4 + lib/ProxySQL_GloVars.cpp | 4 + lib/ProxySQL_HTTP_Server.cpp | 3 + lib/Query_Cache.cpp | 2 +- lib/Query_Processor.cpp | 3 + lib/mysql_connection.cpp | 2 +- lib/sqlite3db.cpp | 2 +- 17 files changed, 159 insertions(+), 304 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 9014767fd1..bfd6f329ed 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -18,6 +18,11 @@ #include "ev.h" +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + #ifdef DEBUG /* */ // Enabling STRESSTEST_POOL ProxySQL will do a lot of loops in the connection pool @@ -570,6 +575,8 @@ class MySQL_HostGroups_Manager { void wrunlock(); int servers_add(SQLite3_result *resultset); bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0); + void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init); + void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName); // used by commit_update_checksums_from_tables() /** * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime. @@ -583,23 +590,13 @@ class MySQL_HostGroups_Manager { * Cluster to propagate current config. * @param The resulset to be stored replacing the current one. */ - void save_incoming_replication_hostgroups(SQLite3_result *); - void save_incoming_group_replication_hostgroups(SQLite3_result *); - void save_incoming_galera_hostgroups(SQLite3_result *); - void save_incoming_aws_aurora_hostgroups(SQLite3_result *); - SQLite3_result* get_current_mysql_servers_inner(); - SQLite3_result* get_current_mysql_replication_hostgroups_inner(); - SQLite3_result* get_current_mysql_group_replication_hostgroups_inner(); - SQLite3_result* get_current_mysql_galera_hostgroups(); - SQLite3_result* get_current_mysql_aws_aurora_hostgroups(); + void save_incoming_mysql_table(SQLite3_result *, const string&); + SQLite3_result* get_current_mysql_table(const string& name); SQLite3_result * execute_query(char *query, char **error); - SQLite3_result *dump_table_mysql_servers(); - SQLite3_result *dump_table_mysql_replication_hostgroups(); - SQLite3_result *dump_table_mysql_group_replication_hostgroups(); - SQLite3_result *dump_table_mysql_galera_hostgroups(); - SQLite3_result *dump_table_mysql_aws_aurora_hostgroups(); + SQLite3_result *dump_table_mysql(const string&); + /** * @brief Update the public member resulset 'mysql_servers_to_monitor'. This resulset should contain the latest * 'mysql_servers' present in 'MySQL_HostGroups_Manager' db, which are not 'OFFLINE_HARD'. The resulset diff --git a/lib/ClickHouse_Authentication.cpp b/lib/ClickHouse_Authentication.cpp index 7bf86dcbb7..1ba0062072 100644 --- a/lib/ClickHouse_Authentication.cpp +++ b/lib/ClickHouse_Authentication.cpp @@ -3,7 +3,10 @@ #include "proxysql.h" #include "cpp.h" #include "proxysql_atomic.h" +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif ClickHouse_Authentication::ClickHouse_Authentication() { #ifdef DEBUG diff --git a/lib/ClickHouse_Server.cpp b/lib/ClickHouse_Server.cpp index b3b61061cf..4fc3d46e3b 100644 --- a/lib/ClickHouse_Server.cpp +++ b/lib/ClickHouse_Server.cpp @@ -23,7 +23,10 @@ #include #include #include +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif #include #include diff --git a/lib/MySQL_Authentication.cpp b/lib/MySQL_Authentication.cpp index 1a463082c9..e06bc8a003 100644 --- a/lib/MySQL_Authentication.cpp +++ b/lib/MySQL_Authentication.cpp @@ -5,7 +5,10 @@ #include "MySQL_Authentication.hpp" +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif MySQL_Authentication::MySQL_Authentication() { #ifdef DEBUG diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index df88ca14f8..c194259f5b 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1,7 +1,6 @@ #include "MySQL_HostGroups_Manager.h" #include "proxysql.h" #include "cpp.h" -#include "SpookyV2.h" #include "MySQL_PreparedStatement.h" #include "MySQL_Data_Stream.h" @@ -1560,6 +1559,34 @@ SQLite3_result * MySQL_HostGroups_Manager::execute_query(char *query, char **err return resultset; } +void MySQL_HostGroups_Manager::CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + string query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); + if (resultset) { + if (resultset->rows_count) { + if (init == false) { + init = true; + myhash.Init(19,3); + } + uint64_t hash1_ = resultset->raw_checksum(); + myhash.Update(&hash1_, sizeof(hash1_)); + proxy_info("Checksum for table %s is %lu\n", TableName.c_str(), hash1_); + } + delete resultset; + } +} + +void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& myhash, bool& init) { + CUCFT1(myhash,init,"mysql_replication_hostgroups","writer_hostgroup"); + CUCFT1(myhash,init,"mysql_group_replication_hostgroups","writer_hostgroup"); + CUCFT1(myhash,init,"mysql_galera_hostgroups","writer_hostgroup"); + CUCFT1(myhash,init,"mysql_aws_aurora_hostgroups","writer_hostgroup"); +} + bool MySQL_HostGroups_Manager::commit( SQLite3_result* runtime_mysql_servers, const std::string& checksum, const time_t epoch ) { @@ -1861,86 +1888,9 @@ bool MySQL_HostGroups_Manager::commit( delete resultset; } } - { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT * FROM mysql_replication_hostgroups ORDER BY writer_hostgroup"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (resultset) { - if (resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = resultset->raw_checksum(); - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is %lu\n", "mysql_replication_hostgroups", hash1_); - } - delete resultset; - } - } - { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT * FROM mysql_group_replication_hostgroups ORDER BY writer_hostgroup"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (resultset) { - if (resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = resultset->raw_checksum(); - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is %lu\n", "mysql_group_replication_hostgroups", hash1_); - } - delete resultset; - } - } - { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT * FROM mysql_galera_hostgroups ORDER BY writer_hostgroup"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (resultset) { - if (resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = resultset->raw_checksum(); - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is %lu\n", "mysql_galera_hostgroups", hash1_); - } - delete resultset; - } - } - { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (resultset) { - if (resultset->rows_count) { - if (init == false) { - init = true; - myhash.Init(19,3); - } - uint64_t hash1_ = resultset->raw_checksum(); - myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is %lu\n", "mysql_aws_aurora_hostgroups", hash1_); - } - delete resultset; - } - } + + commit_update_checksums_from_tables(myhash, init); + if (init == true) { myhash.Final(&hash1, &hash2); } @@ -2481,26 +2431,6 @@ void MySQL_HostGroups_Manager::generate_mysql_galera_hostgroups_table() { pthread_mutex_unlock(&Galera_Info_mutex); } -SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_servers() { - wrlock(); - - // purge table - purge_mysql_servers_table(); - - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); - mydb->execute("DELETE FROM mysql_servers"); - generate_mysql_servers_table(); - - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", MYHGM_GEN_ADMIN_RUNTIME_SERVERS); - mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error , &cols , &affected_rows , &resultset); - wrunlock(); - return resultset; -} - void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) { if (lock) { wrlock(); @@ -2533,58 +2463,39 @@ void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) MySQL_Monitor::trigger_dns_cache_update(); } -SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_replication_hostgroups() { - wrlock(); - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT writer_hostgroup, reader_hostgroup, check_type, comment FROM mysql_replication_hostgroups"; - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - wrunlock(); - return resultset; -} - -SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_group_replication_hostgroups() { - wrlock(); - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_group_replication_hostgroups"; - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - wrunlock(); - return resultset; -} - -SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_galera_hostgroups() { +SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) { + char * query = (char *)""; + if (name == "mysql_aws_aurora_hostgroups") { + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,comment FROM mysql_aws_aurora_hostgroups"; + } else if (name == "mysql_galera_hostgroups") { + query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; + } else if (name == "mysql_group_replication_hostgroups") { + query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_group_replication_hostgroups"; + } else if (name == "mysql_replication_hostgroups") { + query=(char *)"SELECT writer_hostgroup, reader_hostgroup, check_type, comment FROM mysql_replication_hostgroups"; + } else if (name == "mysql_servers") { + query = (char *)MYHGM_GEN_ADMIN_RUNTIME_SERVERS; + } else { + assert(0); + } wrlock(); + if (name == "mysql_servers") { + purge_mysql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + } char *error=NULL; int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); wrunlock(); return resultset; } -SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_aws_aurora_hostgroups() { - wrlock(); - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," - "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,comment FROM mysql_aws_aurora_hostgroups"; - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - wrunlock(); - return resultset; -} MyHGC * MySQL_HostGroups_Manager::MyHGC_create(unsigned int _hid) { MyHGC *myhgc=new MyHGC(_hid); @@ -3735,6 +3646,26 @@ int MySQL_HostGroups_Manager::get_multiple_idle_connections(int _hid, unsigned l return num_conn_current; } +void MySQL_HostGroups_Manager::save_incoming_mysql_table(SQLite3_result *s, const string& name) { + SQLite3_result ** inc = NULL; + if (name == "mysql_aws_aurora_hostgroups") { + inc = &incoming_aws_aurora_hostgroups; + } else if (name == "mysql_galeran_hostgroups") { + inc = &incoming_galera_hostgroups; + } else if (name == "mysql_group_replication_hostgroups") { + inc = &incoming_group_replication_hostgroups; + } else if (name == "mysql_replication_hostgroups") { + inc = &incoming_replication_hostgroups; + } else { + assert(0); + } + if (*inc != nullptr) { + delete *inc; + *inc = nullptr; + } + *inc = s; +} + void MySQL_HostGroups_Manager::save_runtime_mysql_servers(SQLite3_result *s) { if (runtime_mysql_servers) { delete runtime_mysql_servers; @@ -3743,57 +3674,24 @@ void MySQL_HostGroups_Manager::save_runtime_mysql_servers(SQLite3_result *s) { runtime_mysql_servers=s; } -void MySQL_HostGroups_Manager::save_incoming_replication_hostgroups(SQLite3_result *s) { - if (incoming_replication_hostgroups) { - delete incoming_replication_hostgroups; - incoming_replication_hostgroups = nullptr; - } - incoming_replication_hostgroups=s; -} - -void MySQL_HostGroups_Manager::save_incoming_group_replication_hostgroups(SQLite3_result *s) { - if (incoming_group_replication_hostgroups) { - delete incoming_group_replication_hostgroups; - incoming_group_replication_hostgroups = NULL; - } - incoming_group_replication_hostgroups=s; -} - -void MySQL_HostGroups_Manager::save_incoming_galera_hostgroups(SQLite3_result *s) { - if (incoming_galera_hostgroups) { - delete incoming_galera_hostgroups; - incoming_galera_hostgroups = NULL; - } - incoming_galera_hostgroups=s; -} - -void MySQL_HostGroups_Manager::save_incoming_aws_aurora_hostgroups(SQLite3_result *s) { - if (incoming_aws_aurora_hostgroups) { - delete incoming_aws_aurora_hostgroups; - incoming_aws_aurora_hostgroups = NULL; +SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& name) { + if (name == "mysql_aws_aurora_hostgroups") { + return this->incoming_aws_aurora_hostgroups; + } else if (name == "mysql_galera_hostgroups") { + return this->incoming_galera_hostgroups; + } else if (name == "mysql_group_replication_hostgroups") { + return this->incoming_group_replication_hostgroups; + } else if (name == "mysql_replication_hostgroups") { + return this->incoming_replication_hostgroups; + } else if (name == "mysql_serverss") { + return this->runtime_mysql_servers; + } else { + assert(0); } - incoming_aws_aurora_hostgroups=s; -} - -SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_servers_inner() { - return this->runtime_mysql_servers; -} - -SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_replication_hostgroups_inner() { - return this->incoming_replication_hostgroups; + return NULL; } -SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_group_replication_hostgroups_inner() { - return this->incoming_group_replication_hostgroups; -} -SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_galera_hostgroups() { - return this->incoming_galera_hostgroups; -} - -SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_aws_aurora_hostgroups() { - return this->incoming_aws_aurora_hostgroups; -} SQLite3_result * MySQL_HostGroups_Manager::SQL3_Free_Connections() { const int colnum=13; diff --git a/lib/MySQL_PreparedStatement.cpp b/lib/MySQL_PreparedStatement.cpp index 80ed1766bb..a7bc04b7a2 100644 --- a/lib/MySQL_PreparedStatement.cpp +++ b/lib/MySQL_PreparedStatement.cpp @@ -1,7 +1,10 @@ #include "proxysql.h" #include "cpp.h" +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif #include "MySQL_PreparedStatement.h" #include "MySQL_Protocol.h" diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 4a9a7a24aa..c511f68a56 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -5,7 +5,6 @@ #include "proxysql_utils.h" #include "re2/re2.h" #include "re2/regexp.h" -#include "SpookyV2.h" #include "mysqld_error.h" #include "set_parser.h" diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 6c07718122..ba6ed9a071 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -9,7 +9,6 @@ #include "proxysql.h" #include "cpp.h" #include "MySQL_Thread.h" -#include "SpookyV2.h" #include #include #include "re2/re2.h" diff --git a/lib/MySQL_Variables.cpp b/lib/MySQL_Variables.cpp index f0c4111c7a..e60c4b3abb 100644 --- a/lib/MySQL_Variables.cpp +++ b/lib/MySQL_Variables.cpp @@ -3,7 +3,10 @@ #include "MySQL_Session.h" #include "MySQL_Data_Stream.h" +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif #include diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 54f8644a9b..2a7f54debb 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -42,7 +42,10 @@ #include #include #include +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif #include #include @@ -3716,98 +3719,27 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { // handle special queries from Cluster // for bug #1188 , ProxySQL Admin needs to know the exact query - if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - GloAdmin->mysql_servers_wrlock(); - resultset = MyHGM->get_current_mysql_servers_inner(); - GloAdmin->mysql_servers_wrunlock(); - - if (resultset == nullptr) { - resultset=MyHGM->dump_table_mysql_servers(); - if (resultset) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query=false; - goto __run_query; - } - } else { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - run_query=false; - goto __run_query; - } - } - } - - if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { - GloAdmin->mysql_servers_wrlock(); - resultset = MyHGM->get_current_mysql_replication_hostgroups_inner(); - GloAdmin->mysql_servers_wrunlock(); - if (resultset == nullptr) { - resultset=MyHGM->dump_table_mysql_replication_hostgroups(); - if (resultset) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query=false; - goto __run_query; - } - } else { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - run_query=false; - goto __run_query; - } - } - } - if (!strncasecmp(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { - GloAdmin->mysql_servers_wrlock(); - resultset = MyHGM->get_current_mysql_group_replication_hostgroups_inner(); - GloAdmin->mysql_servers_wrunlock(); - if (resultset == nullptr) { - resultset=MyHGM->dump_table_mysql_group_replication_hostgroups(); - if (resultset) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query=false; - goto __run_query; - } - } else { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - run_query=false; - goto __run_query; - } - } - } - - if (!strncasecmp(CLUSTER_QUERY_MYSQL_GALERA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GALERA))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + string tn = ""; + if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS))) { + tn = "mysql_servers"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) { + tn = "mysql_replication_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS))) { + tn = "mysql_group_replication_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_GALERA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GALERA))) { + tn = "mysql_galera_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_AURORA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_AURORA))) { + tn = "mysql_aws_aurora_hostgroups"; + } + if (tn != "") { GloAdmin->mysql_servers_wrlock(); - resultset = MyHGM->get_current_mysql_galera_hostgroups(); + resultset = MyHGM->get_current_mysql_table(tn); GloAdmin->mysql_servers_wrunlock(); - if (resultset == nullptr) { - resultset=MyHGM->dump_table_mysql_galera_hostgroups(); - if (resultset) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query=false; - goto __run_query; - } - } else { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - run_query=false; - goto __run_query; - } - } - } - if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_AURORA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_AURORA))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { - GloAdmin->mysql_servers_wrlock(); - resultset = MyHGM->get_current_mysql_aws_aurora_hostgroups(); - GloAdmin->mysql_servers_wrunlock(); if (resultset == nullptr) { - resultset=MyHGM->dump_table_mysql_aws_aurora_hostgroups(); + resultset=MyHGM->dump_table_mysql(tn); if (resultset) { sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); delete resultset; @@ -3819,6 +3751,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { run_query=false; goto __run_query; } + } } @@ -11676,7 +11609,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { } proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); - resultset=MyHGM->dump_table_mysql_servers(); + resultset=MyHGM->dump_table_mysql("mysql_servers"); if (resultset) { int rc; sqlite3_stmt *statement1=NULL; @@ -11757,7 +11690,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { } proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); - resultset=MyHGM->dump_table_mysql_replication_hostgroups(); + resultset=MyHGM->dump_table_mysql("mysql_replication_hostgroups"); if (resultset) { for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; @@ -11795,7 +11728,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { } proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); - resultset=MyHGM->dump_table_mysql_group_replication_hostgroups(); + resultset=MyHGM->dump_table_mysql("mysql__group_replication_hostgroups"); if (resultset) { int rc; sqlite3_stmt *statement=NULL; @@ -11839,7 +11772,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { } proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); - resultset=MyHGM->dump_table_mysql_galera_hostgroups(); + resultset=MyHGM->dump_table_mysql("mysql_galera_hostgroups"); if (resultset) { int rc; sqlite3_stmt *statement=NULL; @@ -11884,7 +11817,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { } proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); - resultset=MyHGM->dump_table_mysql_aws_aurora_hostgroups(); + resultset=MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); if (resultset) { int rc; sqlite3_stmt *statement=NULL; @@ -12004,7 +11937,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->save_incoming_replication_hostgroups(resultset_replication); + MyHGM->save_incoming_mysql_table(resultset_replication,"mysql_replication_hostgroups"); } //if (resultset) delete resultset; //resultset=NULL; @@ -12038,7 +11971,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->save_incoming_group_replication_hostgroups(resultset_group_replication); + MyHGM->save_incoming_mysql_table(resultset_group_replication,"mysql_group_replication_hostgroups"); } // support for Galera, table mysql_galera_hostgroups @@ -12069,7 +12002,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->save_incoming_galera_hostgroups(resultset_galera); + MyHGM->save_incoming_mysql_table(resultset_galera, "mysql_galera_hostgroups"); } // support for AWS Aurora, table mysql_aws_aurora_hostgroups @@ -12104,7 +12037,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->save_incoming_aws_aurora_hostgroups(resultset_aws_aurora); + MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); } // commit all the changes MyHGM->commit(runtime_mysql_servers, checksum, epoch); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 6e6d29877f..1e6260e961 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -3,7 +3,11 @@ #include "proxysql.h" #include "proxysql_utils.h" #include "cpp.h" +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif + #include "prometheus_helpers.h" #include "ProxySQL_Cluster.hpp" diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index 741f751dde..1df01f5313 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -4,7 +4,11 @@ #include #include #include +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif + #include #include diff --git a/lib/ProxySQL_HTTP_Server.cpp b/lib/ProxySQL_HTTP_Server.cpp index c116a12c9a..e82b18a249 100644 --- a/lib/ProxySQL_HTTP_Server.cpp +++ b/lib/ProxySQL_HTTP_Server.cpp @@ -22,7 +22,10 @@ #include #include #include +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif #include #include diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index a6af25a47b..59315e2d6a 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -4,7 +4,7 @@ #include "cpp.h" #include "query_cache.hpp" #include "proxysql_atomic.h" -#include "SpookyV2.h" +//#include "SpookyV2.h" #include "prometheus_helpers.h" #include "MySQL_Protocol.h" diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 1382cb4cfe..debba9d220 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -10,7 +10,10 @@ #include "MySQL_Data_Stream.h" #include "query_processor.h" +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif #include "pcrecpp.h" diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index acc06ba981..7b9944dc0b 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -1,7 +1,7 @@ #include "MySQL_HostGroups_Manager.h" #include "proxysql.h" #include "cpp.h" -#include "SpookyV2.h" +//#include "SpookyV2.h" #include #include diff --git a/lib/sqlite3db.cpp b/lib/sqlite3db.cpp index e839ef2110..4b715aac8f 100644 --- a/lib/sqlite3db.cpp +++ b/lib/sqlite3db.cpp @@ -1,6 +1,6 @@ #include "proxysql.h" #include "cpp.h" -#include "SpookyV2.h" +//#include "SpookyV2.h" #include #include #include From 107a864521c689e3ce53e6042e10cc032860586e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 12 Jan 2023 07:49:35 +0000 Subject: [PATCH 03/13] Fix compiling --- src/SQLite3_Server.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 3c04a3f742..8e710481a8 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -24,7 +24,11 @@ #include #include #include +#include +#ifndef SPOOKYV2 #include "SpookyV2.h" +#define SPOOKYV2 +#endif #include #include From 36575d273ac8606eca32f13fc7a10750a187e1bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Jan 2023 20:37:15 +0000 Subject: [PATCH 04/13] Several code improvements In Admin: - replaced a lot of flush_[...]__from_memory_to_disk() and flush_[...]__from_disk_to_memory() with a generic flush_GENERIC__from_to() - several commands like "LOAD MYSQL SERVERS FROM DISK" and similar are automatically generated for various modules (not all) and saved/map in a map load_save_disk_commands - FlushCommandWrapper() is able to map the various LOAD/SAVE from/to DISK into commands to run on SQLite --- include/proxysql_admin.h | 13 +- lib/MySQL_HostGroups_Manager.cpp | 2 +- lib/ProxySQL_Admin.cpp | 450 +++++++++++-------------------- lib/ProxySQL_Cluster.cpp | 6 +- 4 files changed, 164 insertions(+), 307 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 5ba16f335a..149b69cf49 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -357,14 +357,11 @@ class ProxySQL_Admin { int load_debug_to_runtime(); void save_debug_from_runtime(); #endif // DEBUG + + void flush_GENERIC__from_to(const std::string&, const std::string&); + void flush_mysql_users__from_memory_to_disk(); void flush_mysql_users__from_disk_to_memory(); - void flush_mysql_servers__from_memory_to_disk(); - void flush_mysql_servers__from_disk_to_memory(); - void flush_mysql_query_rules__from_memory_to_disk(); - void flush_mysql_query_rules__from_disk_to_memory(); - void flush_mysql_firewall__from_memory_to_disk(); - void flush_mysql_firewall__from_disk_to_memory(); // void flush_mysql_variables__from_disk_to_memory(); // commented in 2.3 because unused void flush_mysql_variables__from_memory_to_disk(); @@ -408,8 +405,6 @@ class ProxySQL_Admin { void load_scheduler_to_runtime(); void save_scheduler_runtime_to_database(bool); - void flush_scheduler__from_memory_to_disk(); - void flush_scheduler__from_disk_to_memory(); void load_admin_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0) { flush_admin_variables___database_to_runtime(admindb, true, checksum, epoch); } void save_admin_variables_from_runtime() { flush_admin_variables___runtime_to_database(admindb, true, true, false); } @@ -463,8 +458,6 @@ class ProxySQL_Admin { // Cluster void load_proxysql_servers_to_runtime(bool _lock=true, const std::string& checksum = "", const time_t epoch = 0); - void flush_proxysql_servers__from_memory_to_disk(); - void flush_proxysql_servers__from_disk_to_memory(); void save_proxysql_servers_runtime_to_database(bool); void dump_checksums_values_table(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c194259f5b..766e4ef35c 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3650,7 +3650,7 @@ void MySQL_HostGroups_Manager::save_incoming_mysql_table(SQLite3_result *s, cons SQLite3_result ** inc = NULL; if (name == "mysql_aws_aurora_hostgroups") { inc = &incoming_aws_aurora_hostgroups; - } else if (name == "mysql_galeran_hostgroups") { + } else if (name == "mysql_galera_hostgroups") { inc = &incoming_galera_hostgroups; } else if (name == "mysql_group_replication_hostgroups") { inc = &incoming_group_replication_hostgroups; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 2a7f54debb..1ed7de1733 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -98,6 +98,47 @@ char * proxysql_version = NULL; MARIADB_CHARSET_INFO * proxysql_find_charset_name(const char *name); + +static const vector mysql_servers_tablenames = { + "mysql_servers", + "mysql_replication_hostgroups", + "mysql_group_replication_hostgroups", + "mysql_galera_hostgroups", + "mysql_aws_aurora_hostgroups", +}; + +static const vector mysql_firewall_tablenames = { + "mysql_firewall_whitelist_users", + "mysql_firewall_whitelist_rules", + "mysql_firewall_whitelist_sqli_fingerprints", +}; + +static const vector mysql_query_rules_tablenames = { "mysql_query_rules", "mysql_query_rules_fast_routing" }; +static const vector scheduler_tablenames = { "scheduler" }; +static const vector proxysql_servers_tablenames = { "proxysql_servers" }; + +static unordered_map&> module_tablenames = { + { "mysql_servers", mysql_servers_tablenames }, + { "mysql_firewall", mysql_firewall_tablenames }, + { "mysql_query_rules", mysql_query_rules_tablenames }, + { "scheduler", scheduler_tablenames }, + { "proxysql_servers", proxysql_servers_tablenames }, +}; + +static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, const string& p2, const string& p3) { + string query; + for (auto it = tbs.begin(); it != tbs.end(); it++) { + if (p1 != "") { + query = p1 + *it; + db->execute(query.c_str()); + } + if (p2 != "" && p3 != "") { + query = p2 + *it + p3 + *it; + db->execute(query.c_str()); + } + } +} + /* static long get_file_size (const char *filename) { @@ -904,16 +945,6 @@ const std::vector SAVE_ADMIN_VARIABLES_TO_MEMORY = { "SAVE ADMIN VARIABLES FROM RUNTIME" , "SAVE ADMIN VARIABLES FROM RUN" }; -const std::vector LOAD_MYSQL_SERVERS_TO_MEMORY = { - "LOAD MYSQL SERVERS TO MEMORY" , - "LOAD MYSQL SERVERS TO MEM" , - "LOAD MYSQL SERVERS FROM DISK" }; - -const std::vector SAVE_MYSQL_SERVERS_FROM_MEMORY = { - "SAVE MYSQL SERVERS FROM MEMORY" , - "SAVE MYSQL SERVERS FROM MEM" , - "SAVE MYSQL SERVERS TO DISK" }; - const std::vector LOAD_MYSQL_SERVERS_FROM_MEMORY = { "LOAD MYSQL SERVERS FROM MEMORY" , "LOAD MYSQL SERVERS FROM MEM" , @@ -926,16 +957,6 @@ const std::vector SAVE_MYSQL_SERVERS_TO_MEMORY = { "SAVE MYSQL SERVERS FROM RUNTIME" , "SAVE MYSQL SERVERS FROM RUN" }; -const std::vector LOAD_MYSQL_USERS_TO_MEMORY = { - "LOAD MYSQL USERS TO MEMORY" , - "LOAD MYSQL USERS TO MEM" , - "LOAD MYSQL USERS FROM DISK" }; - -const std::vector SAVE_MYSQL_USERS_FROM_MEMORY = { - "SAVE MYSQL USERS FROM MEMORY" , - "SAVE MYSQL USERS FROM MEM" , - "SAVE MYSQL USERS TO DISK" }; - const std::vector LOAD_MYSQL_USERS_FROM_MEMORY = { "LOAD MYSQL USERS FROM MEMORY" , "LOAD MYSQL USERS FROM MEM" , @@ -948,16 +969,6 @@ const std::vector SAVE_MYSQL_USERS_TO_MEMORY = { "SAVE MYSQL USERS FROM RUNTIME" , "SAVE MYSQL USERS FROM RUN" }; -const std::vector LOAD_MYSQL_VARIABLES_TO_MEMORY = { - "LOAD MYSQL VARIABLES TO MEMORY" , - "LOAD MYSQL VARIABLES TO MEM" , - "LOAD MYSQL VARIABLES FROM DISK" }; - -const std::vector SAVE_MYSQL_VARIABLES_FROM_MEMORY = { - "SAVE MYSQL VARIABLES FROM MEMORY" , - "SAVE MYSQL VARIABLES FROM MEM" , - "SAVE MYSQL VARIABLES TO DISK" }; - const std::vector LOAD_MYSQL_VARIABLES_FROM_MEMORY = { "LOAD MYSQL VARIABLES FROM MEMORY" , "LOAD MYSQL VARIABLES FROM MEM" , @@ -971,6 +982,31 @@ const std::vector SAVE_MYSQL_VARIABLES_TO_MEMORY = { "SAVE MYSQL VARIABLES FROM RUN" }; +static unordered_map, vector>> load_save_disk_commands; + +static void generate_load_save_disk_commands(std::vector& vec1, std::vector& vec2, const string& name) { + string s; + if (vec1.size() == 0) { + s = "LOAD " + name + " TO MEMORY"; vec1.push_back(s); + s = "LOAD " + name + " TO MEM"; vec1.push_back(s); + s = "LOAD " + name + " FROM DISK"; vec1.push_back(s); + } + if (vec2.size() == 0) { + s = "SAVE " + name + " FROM MEMORY"; vec2.push_back(s); + s = "SAVE " + name + " FROM MEM"; vec2.push_back(s); + s = "SAVE " + name + " TO DISK"; vec2.push_back(s); + } +} + +static void generate_load_save_disk_commands(const string& name, const string& command) { + std::vector vec1; + std::vector vec2; + generate_load_save_disk_commands(vec1, vec2, name); + std::tuple, vector> a = tuple, vector>{command, vec1, vec2}; + load_save_disk_commands[name] = a; +} + + bool is_admin_command_or_alias(const std::vector& cmds, char *query_no_space, int query_no_space_length) { for (std::vector::const_iterator it=cmds.begin(); it!=cmds.end(); ++it) { if ((unsigned int)query_no_space_length==it->length() && !strncasecmp(it->c_str(), query_no_space, query_no_space_length)) { @@ -981,6 +1017,36 @@ bool is_admin_command_or_alias(const std::vector& cmds, char *query return false; } +bool FlushCommandWrapper(MySQL_Session *sess, const std::vector& cmds, char *query_no_space, int query_no_space_length, const string& name, const string& direction) { + if ( is_admin_command_or_alias(cmds, query_no_space, query_no_space_length) ) { + ProxySQL_Admin *SPA = GloAdmin; + SPA->flush_GENERIC__from_to(name, direction); +#ifdef DEBUG + string msg = "Loaded " + name + " "; + if (direction == "memory_to_disk") + msg += "from MEMORY to DISK"; + else if (direction == "disk_to_memory") + msg += "from DISK to MEMORY"; + else + assert(0); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers to MEMORY\n"); +#endif // DEBUG + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return true; + } + return false; +} + +bool FlushCommandWrapper(MySQL_Session *sess, const string& modname, char *query_no_space, int query_no_space_length) { + assert(load_save_disk_commands.find(modname) != load_save_disk_commands.end()); + tuple, vector>& t = load_save_disk_commands[modname]; + if (FlushCommandWrapper(sess, get<1>(t), query_no_space, query_no_space_length, modname, "disk_to_memory") == true) + return true; + if (FlushCommandWrapper(sess, get<2>(t), query_no_space, query_no_space_length, modname, "memory_to_disk") == true) + return true; + return false; +} + incoming_servers_t::incoming_servers_t() {} incoming_servers_t::incoming_servers_t( @@ -2031,35 +2097,8 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query if ((query_no_space_length>15) && ( (!strncasecmp("SAVE SCHEDULER ", query_no_space, 15)) || (!strncasecmp("LOAD SCHEDULER ", query_no_space, 15))) ) { - if ( - (query_no_space_length==strlen("LOAD SCHEDULER TO MEMORY") && !strncasecmp("LOAD SCHEDULER TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SCHEDULER TO MEM") && !strncasecmp("LOAD SCHEDULER TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SCHEDULER FROM DISK") && !strncasecmp("LOAD SCHEDULER FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_scheduler__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading scheduler to to MEMORY\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + if (FlushCommandWrapper(sess, "scheduler", query_no_space, query_no_space_length) == true) return false; - } - - if ( - (query_no_space_length==strlen("SAVE SCHEDULER FROM MEMORY") && !strncasecmp("SAVE SCHEDULER FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SCHEDULER FROM MEM") && !strncasecmp("SAVE SCHEDULER FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SCHEDULER TO DISK") && !strncasecmp("SAVE SCHEDULER TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_scheduler__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving scheduler to DISK\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } if ( (query_no_space_length==strlen("LOAD SCHEDULER FROM MEMORY") && !strncasecmp("LOAD SCHEDULER FROM MEMORY",query_no_space, query_no_space_length)) @@ -2250,22 +2289,22 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query } if ((query_no_space_length>17) && ( (!strncasecmp("SAVE MYSQL USERS ", query_no_space, 17)) || (!strncasecmp("LOAD MYSQL USERS ", query_no_space, 17))) ) { - if ( is_admin_command_or_alias(LOAD_MYSQL_USERS_TO_MEMORY, query_no_space, query_no_space_length) ) { + string modname = "mysql_users"; + tuple, vector>& t = load_save_disk_commands[modname]; + if ( is_admin_command_or_alias(get<1>(t), query_no_space, query_no_space_length) ) { ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; SPA->flush_mysql_users__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading mysql users to MEMORY\n"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading %s to MEMORY\n", modname.c_str()); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return false; } - - if ( is_admin_command_or_alias(SAVE_MYSQL_USERS_FROM_MEMORY, query_no_space, query_no_space_length) ) { + if ( is_admin_command_or_alias(get<2>(t), query_no_space, query_no_space_length) ) { ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; SPA->flush_mysql_users__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving mysql users to DISK\n"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving %s to DISK\n", modname.c_str()); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return false; } - if ( is_admin_command_or_alias(LOAD_MYSQL_USERS_FROM_MEMORY, query_no_space, query_no_space_length) ) { ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; SPA->init_users(); @@ -2511,14 +2550,16 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query if ((query_no_space_length>21) && ( (!strncasecmp("SAVE MYSQL VARIABLES ", query_no_space, 21)) || (!strncasecmp("LOAD MYSQL VARIABLES ", query_no_space, 21))) ) { - if ( is_admin_command_or_alias(LOAD_MYSQL_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length) ) { + string modname = "mysql_variables"; + tuple, vector>& t = load_save_disk_commands[modname]; + if ( is_admin_command_or_alias(get<1>(t), query_no_space, query_no_space_length) ) { l_free(*ql,*q); *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'mysql-%'"); *ql=strlen(*q)+1; return true; } - if ( is_admin_command_or_alias(SAVE_MYSQL_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length) ) { + if ( is_admin_command_or_alias(get<2>(t), query_no_space, query_no_space_length) ) { l_free(*ql,*q); *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'mysql-%'"); *ql=strlen(*q)+1; @@ -2573,21 +2614,8 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query if ((query_no_space_length>19) && ( (!strncasecmp("SAVE MYSQL SERVERS ", query_no_space, 19)) || (!strncasecmp("LOAD MYSQL SERVERS ", query_no_space, 19))) ) { - if ( is_admin_command_or_alias(LOAD_MYSQL_SERVERS_TO_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_servers__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers to MEMORY\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( is_admin_command_or_alias(SAVE_MYSQL_SERVERS_FROM_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_servers__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql servers to DISK\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + if (FlushCommandWrapper(sess, "mysql_servers", query_no_space, query_no_space_length) == true) return false; - } if ( is_admin_command_or_alias(LOAD_MYSQL_SERVERS_FROM_MEMORY, query_no_space, query_no_space_length) ) { ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; @@ -2640,35 +2668,17 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query if ((query_no_space_length>22) && ( (!strncasecmp("SAVE PROXYSQL SERVERS ", query_no_space, 22)) || (!strncasecmp("LOAD PROXYSQL SERVERS ", query_no_space, 22))) ) { - if ( - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS TO MEMORY") && !strncasecmp("LOAD PROXYSQL SERVERS TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS TO MEM") && !strncasecmp("LOAD PROXYSQL SERVERS TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM DISK") && !strncasecmp("LOAD PROXYSQL SERVERS FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_proxysql_servers__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ProxySQL servers to MEMORY\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + if (FlushCommandWrapper(sess, "proxysql_servers", query_no_space, query_no_space_length) == true) return false; - } - if ( - (query_no_space_length==strlen("SAVE PROXYSQL SERVERS FROM MEMORY") && !strncasecmp("SAVE PROXYSQL SERVERS FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE PROXYSQL SERVERS FROM MEM") && !strncasecmp("SAVE PROXYSQL SERVERS FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE PROXYSQL SERVERS TO DISK") && !strncasecmp("SAVE PROXYSQL SERVERS TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_proxysql_servers__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved ProxySQL servers to DISK\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); +/* + string modname = "proxysql_servers"; + tuple, vector>& t = load_save_disk_commands[modname]; + if (FlushCommandWrapper(sess, get<1>(t), query_no_space, query_no_space_length, modname, "disk_to_memory") == true) return false; - } + if (FlushCommandWrapper(sess, get<2>(t), query_no_space, query_no_space_length, modname, "memory_to_disk") == true) + return false; +*/ if ( (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM MEMORY") && !strncasecmp("LOAD PROXYSQL SERVERS FROM MEMORY",query_no_space, query_no_space_length)) || @@ -2754,20 +2764,8 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query if ((query_no_space_length>20) && ( (!strncasecmp("SAVE MYSQL FIREWALL ", query_no_space, 20)) || (!strncasecmp("LOAD MYSQL FIREWALL ", query_no_space, 20))) ) { - if ( - (query_no_space_length==strlen("LOAD MYSQL FIREWALL TO MEMORY") && !strncasecmp("LOAD MYSQL FIREWALL TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL FIREWALL TO MEM") && !strncasecmp("LOAD MYSQL FIREWALL TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM DISK") && !strncasecmp("LOAD MYSQL FIREWALL FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_firewall__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql firewall to MEMORY\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + if (FlushCommandWrapper(sess, "mysql_firewall", query_no_space, query_no_space_length) == true) return false; - } if ( (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM CONFIG") && !strncasecmp("LOAD MYSQL FIREWALL FROM CONFIG",query_no_space, query_no_space_length)) @@ -2798,21 +2796,6 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query return false; } - if ( - (query_no_space_length==strlen("SAVE MYSQL FIREWALL FROM MEMORY") && !strncasecmp("SAVE MYSQL FIREWALL FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL FIREWALL FROM MEM") && !strncasecmp("SAVE MYSQL FIREWALL FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL FIREWALL TO DISK") && !strncasecmp("SAVE MYSQL FIREWALL TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_firewall__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql firewall to DISK\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if ( (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM MEMORY") && !strncasecmp("LOAD MYSQL FIREWALL FROM MEMORY",query_no_space, query_no_space_length)) || @@ -2854,20 +2837,8 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query if ((query_no_space_length>23) && ( (!strncasecmp("SAVE MYSQL QUERY RULES ", query_no_space, 23)) || (!strncasecmp("LOAD MYSQL QUERY RULES ", query_no_space, 23))) ) { - if ( - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES TO MEMORY") && !strncasecmp("LOAD MYSQL QUERY RULES TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES TO MEM") && !strncasecmp("LOAD MYSQL QUERY RULES TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM DISK") && !strncasecmp("LOAD MYSQL QUERY RULES FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_query_rules__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql query rules to MEMORY\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + if (FlushCommandWrapper(sess, "mysql_query_rules", query_no_space, query_no_space_length) == true) return false; - } if ( (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM CONFIG") && !strncasecmp("LOAD MYSQL QUERY RULES FROM CONFIG",query_no_space, query_no_space_length)) @@ -2897,21 +2868,6 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query return false; } - if ( - (query_no_space_length==strlen("SAVE MYSQL QUERY RULES FROM MEMORY") && !strncasecmp("SAVE MYSQL QUERY RULES FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL QUERY RULES FROM MEM") && !strncasecmp("SAVE MYSQL QUERY RULES FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL QUERY RULES TO DISK") && !strncasecmp("SAVE MYSQL QUERY RULES TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_query_rules__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql query rules to DISK\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if ( (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM MEMORY") && !strncasecmp("LOAD MYSQL QUERY RULES FROM MEMORY",query_no_space, query_no_space_length)) || @@ -5743,6 +5699,25 @@ ProxySQL_Admin::ProxySQL_Admin() : pthread_mutex_init(&sql_query_global_mutex, NULL); + generate_load_save_disk_commands("mysql_firewall", "MYSQL FIREWALL"); + generate_load_save_disk_commands("mysql_query_rules", "MYSQL QUERY RULES"); + generate_load_save_disk_commands("mysql_users", "MYSQL USERS"); + generate_load_save_disk_commands("mysql_servers", "MYSQL SERVERS"); + generate_load_save_disk_commands("mysql_variables", "MYSQL VARIABLES"); + generate_load_save_disk_commands("scheduler", "SCHEDULER"); + generate_load_save_disk_commands("proxysql_servers", "PROXYSQL SERVERS"); + + { + // we perform some sanity check + assert(load_save_disk_commands.size() > 0); + for (auto it = load_save_disk_commands.begin(); it != load_save_disk_commands.end(); it++) { + vector& vec1 = get<1>(it->second); + assert(vec1.size() == 3); + vector& vec2 = get<2>(it->second); + assert(vec2.size() == 3); + } + } + variables.admin_credentials=strdup("admin:admin"); variables.stats_credentials=strdup("stats:stats"); @@ -10267,17 +10242,10 @@ void ProxySQL_Admin::__insert_or_ignore_maintable_select_disktable() { void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_servers SELECT * FROM disk.mysql_servers"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_replication_hostgroups SELECT * FROM disk.mysql_replication_hostgroups"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_group_replication_hostgroups SELECT * FROM disk.mysql_group_replication_hostgroups"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_galera_hostgroups SELECT * FROM disk.mysql_galera_hostgroups"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_aws_aurora_hostgroups SELECT * FROM disk.mysql_aws_aurora_hostgroups"); + BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); admindb->execute("INSERT OR REPLACE INTO main.mysql_users SELECT * FROM disk.mysql_users"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_query_rules SELECT * FROM disk.mysql_query_rules"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_query_rules_fast_routing SELECT * FROM disk.mysql_query_rules_fast_routing"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_firewall_whitelist_users SELECT * FROM disk.mysql_firewall_whitelist_users"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_firewall_whitelist_rules SELECT * FROM disk.mysql_firewall_whitelist_rules"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_firewall_whitelist_sqli_fingerprints SELECT * FROM disk.mysql_firewall_whitelist_sqli_fingerprints"); + BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); { // online upgrade of mysql-session_idle_ms char *error=NULL; @@ -10303,7 +10271,7 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { if (resultset) delete resultset; } admindb->execute("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables"); - admindb->execute("INSERT OR REPLACE INTO main.scheduler SELECT * FROM disk.scheduler"); + BQE1(admindb, scheduler_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); admindb->execute("INSERT OR REPLACE INTO main.restapi_routes SELECT * FROM disk.restapi_routes"); admindb->execute("INSERT OR REPLACE INTO main.proxysql_servers SELECT * FROM disk.proxysql_servers"); #ifdef DEBUG @@ -10353,19 +10321,12 @@ void ProxySQL_Admin::__delete_disktable() { */ void ProxySQL_Admin::__insert_or_replace_disktable_select_maintable() { - admindb->execute("INSERT OR REPLACE INTO disk.mysql_servers SELECT * FROM main.mysql_servers"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_replication_hostgroups SELECT * FROM main.mysql_replication_hostgroups"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_group_replication_hostgroups SELECT * FROM main.mysql_group_replication_hostgroups"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_galera_hostgroups SELECT * FROM main.mysql_galera_hostgroups"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_aws_aurora_hostgroups SELECT * FROM main.mysql_aws_aurora_hostgroups"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_query_rules SELECT * FROM main.mysql_query_rules"); + BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); admindb->execute("INSERT OR REPLACE INTO disk.mysql_users SELECT * FROM main.mysql_users"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_query_rules_fast_routing SELECT * FROM main.mysql_query_rules_fast_routing"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_firewall_whitelist_users SELECT * FROM main.mysql_firewall_whitelist_users"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_firewall_whitelist_rules SELECT * FROM main.mysql_firewall_whitelist_rules"); - admindb->execute("INSERT OR REPLACE INTO disk.mysql_firewall_whitelist_sqli_fingerprints SELECT * FROM main.mysql_firewall_whitelist_sqli_fingerprints"); + BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); admindb->execute("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables"); - admindb->execute("INSERT OR REPLACE INTO disk.scheduler SELECT * FROM main.scheduler"); + BQE1(admindb, scheduler_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); admindb->execute("INSERT OR REPLACE INTO disk.proxysql_servers SELECT * FROM main.proxysql_servers"); #ifdef DEBUG admindb->execute("INSERT OR REPLACE INTO disk.debug_levels SELECT * FROM main.debug_levels"); @@ -10428,98 +10389,19 @@ void ProxySQL_Admin::flush_clickhouse_users__from_memory_to_disk() { } #endif /* PROXYSQLCLICKHOUSE */ -void ProxySQL_Admin::flush_scheduler__from_disk_to_memory() { - admindb->wrlock(); - admindb->execute("DELETE FROM main.scheduler"); - admindb->execute("INSERT INTO main.scheduler SELECT * FROM disk.scheduler"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_scheduler__from_memory_to_disk() { - admindb->wrlock(); - admindb->execute("DELETE FROM disk.scheduler"); - admindb->execute("INSERT INTO disk.scheduler SELECT * FROM main.scheduler"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_mysql_servers__from_disk_to_memory() { - admindb->wrlock(); - admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM main.mysql_servers"); - admindb->execute("DELETE FROM main.mysql_replication_hostgroups"); - admindb->execute("DELETE FROM main.mysql_group_replication_hostgroups"); - admindb->execute("DELETE FROM main.mysql_galera_hostgroups"); - admindb->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); - admindb->execute("INSERT INTO main.mysql_servers SELECT * FROM disk.mysql_servers"); - admindb->execute("INSERT INTO main.mysql_replication_hostgroups SELECT * FROM disk.mysql_replication_hostgroups"); - admindb->execute("INSERT INTO main.mysql_group_replication_hostgroups SELECT * FROM disk.mysql_group_replication_hostgroups"); - admindb->execute("INSERT INTO main.mysql_galera_hostgroups SELECT * FROM disk.mysql_galera_hostgroups"); - admindb->execute("INSERT INTO main.mysql_aws_aurora_hostgroups SELECT * FROM disk.mysql_aws_aurora_hostgroups"); - admindb->execute("PRAGMA foreign_keys = ON"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_mysql_servers__from_memory_to_disk() { +void ProxySQL_Admin::flush_GENERIC__from_to(const string& name, const string& direction) { + assert(direction == "disk_to_memory" || direction == "memory_to_disk"); admindb->wrlock(); admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM disk.mysql_servers"); - admindb->execute("DELETE FROM disk.mysql_replication_hostgroups"); - admindb->execute("DELETE FROM disk.mysql_group_replication_hostgroups"); - admindb->execute("DELETE FROM disk.mysql_galera_hostgroups"); - admindb->execute("DELETE FROM disk.mysql_aws_aurora_hostgroups"); - admindb->execute("INSERT INTO disk.mysql_servers SELECT * FROM main.mysql_servers"); - admindb->execute("INSERT INTO disk.mysql_replication_hostgroups SELECT * FROM main.mysql_replication_hostgroups"); - admindb->execute("INSERT INTO disk.mysql_group_replication_hostgroups SELECT * FROM main.mysql_group_replication_hostgroups"); - admindb->execute("INSERT INTO disk.mysql_galera_hostgroups SELECT * FROM main.mysql_galera_hostgroups"); - admindb->execute("INSERT INTO disk.mysql_aws_aurora_hostgroups SELECT * FROM main.mysql_aws_aurora_hostgroups"); - admindb->execute("PRAGMA foreign_keys = ON"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_mysql_firewall__from_disk_to_memory() { - admindb->wrlock(); - admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM main.mysql_firewall_whitelist_rules"); - admindb->execute("INSERT INTO main.mysql_firewall_whitelist_rules SELECT * FROM disk.mysql_firewall_whitelist_rules"); - admindb->execute("DELETE FROM main.mysql_firewall_whitelist_users"); - admindb->execute("INSERT INTO main.mysql_firewall_whitelist_users SELECT * FROM disk.mysql_firewall_whitelist_users"); - admindb->execute("DELETE FROM main.mysql_firewall_whitelist_sqli_fingerprints"); - admindb->execute("INSERT INTO main.mysql_firewall_whitelist_sqli_fingerprints SELECT * FROM disk.mysql_firewall_whitelist_sqli_fingerprints"); - admindb->execute("PRAGMA foreign_keys = ON"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_mysql_firewall__from_memory_to_disk() { - admindb->wrlock(); - admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM disk.mysql_firewall_whitelist_rules"); - admindb->execute("INSERT INTO disk.mysql_firewall_whitelist_rules SELECT * FROM main.mysql_firewall_whitelist_rules"); - admindb->execute("DELETE FROM disk.mysql_firewall_whitelist_users"); - admindb->execute("INSERT INTO disk.mysql_firewall_whitelist_users SELECT * FROM main.mysql_firewall_whitelist_users"); - admindb->execute("DELETE FROM disk.mysql_firewall_whitelist_sqli_fingerprints"); - admindb->execute("INSERT INTO disk.mysql_firewall_whitelist_sqli_fingerprints SELECT * FROM main.mysql_firewall_whitelist_sqli_fingerprints"); - admindb->execute("PRAGMA foreign_keys = ON"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_mysql_query_rules__from_disk_to_memory() { - admindb->wrlock(); - admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM main.mysql_query_rules"); - admindb->execute("INSERT INTO main.mysql_query_rules SELECT * FROM disk.mysql_query_rules"); - admindb->execute("DELETE FROM main.mysql_query_rules_fast_routing"); - admindb->execute("INSERT INTO main.mysql_query_rules_fast_routing SELECT * FROM disk.mysql_query_rules_fast_routing"); - admindb->execute("PRAGMA foreign_keys = ON"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_mysql_query_rules__from_memory_to_disk() { - admindb->wrlock(); - admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM disk.mysql_query_rules"); - admindb->execute("INSERT INTO disk.mysql_query_rules SELECT * FROM main.mysql_query_rules"); - admindb->execute("DELETE FROM disk.mysql_query_rules_fast_routing"); - admindb->execute("INSERT INTO disk.mysql_query_rules_fast_routing SELECT * FROM main.mysql_query_rules_fast_routing"); + auto it = module_tablenames.find(name); + assert(it != module_tablenames.end()); + if (direction == "disk_to_memory") { + BQE1(admindb, it->second, "DELETE FROM main.", "INSERT INTO main.", " SELECT * FROM disk."); + } else if (direction == "memory_to_disk") { + BQE1(admindb, it->second, "DELETE FROM disk.", "INSERT INTO disk.", " SELECT * FROM man."); + } else { + assert(0); + } admindb->execute("PRAGMA foreign_keys = ON"); admindb->wrunlock(); } @@ -11728,7 +11610,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { } proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); - resultset=MyHGM->dump_table_mysql("mysql__group_replication_hostgroups"); + resultset=MyHGM->dump_table_mysql("mysql_group_replication_hostgroups"); if (resultset) { int rc; sqlite3_stmt *statement=NULL; @@ -13140,24 +13022,6 @@ void ProxySQL_Admin::load_proxysql_servers_to_runtime(bool _lock, const std::str resultset=NULL; } -void ProxySQL_Admin::flush_proxysql_servers__from_memory_to_disk() { - admindb->wrlock(); - admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM disk.proxysql_servers"); - admindb->execute("INSERT INTO disk.proxysql_servers SELECT * FROM main.proxysql_servers"); - admindb->execute("PRAGMA foreign_keys = ON"); - admindb->wrunlock(); -} - -void ProxySQL_Admin::flush_proxysql_servers__from_disk_to_memory() { - admindb->wrlock(); - admindb->execute("PRAGMA foreign_keys = OFF"); - admindb->execute("DELETE FROM main.proxysql_servers"); - admindb->execute("INSERT INTO main.proxysql_servers SELECT * FROM disk.proxysql_servers"); - admindb->execute("PRAGMA foreign_keys = ON"); - admindb->wrunlock(); -} - void ProxySQL_Admin::save_proxysql_servers_runtime_to_database(bool _runtime) { std::lock_guard lock(proxysql_servers_mutex); // make sure that the caller has called mysql_servers_wrlock() diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 1e6260e961..f6dda66549 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1046,7 +1046,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c ); if (GloProxyCluster->cluster_mysql_query_rules_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); - GloAdmin->flush_mysql_query_rules__from_memory_to_disk(); + GloAdmin->flush_GENERIC__from_to("mysql_query_rules", "memory_to_disk"); } else { proxy_info("Cluster: NOT saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); } @@ -1774,7 +1774,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, GloAdmin->load_mysql_servers_to_runtime(incoming_servers, checksum, epoch); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->flush_mysql_servers__from_memory_to_disk(); + GloAdmin->flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); } else { proxy_info("Cluster: Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); } @@ -2077,7 +2077,7 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect GloAdmin->load_proxysql_servers_to_runtime(false, expected_checksum, epoch); if (GloProxyCluster->cluster_proxysql_servers_save_to_disk == true) { proxy_info("Cluster: Saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->flush_proxysql_servers__from_memory_to_disk(); + GloAdmin->flush_GENERIC__from_to("proxysql_servers","memory_to_disk"); } else { proxy_info("Cluster: NOT saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); } From ceb5695cd2858bc992c8dfb2b70bbcd2a963bb8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Jan 2023 22:42:23 +0000 Subject: [PATCH 05/13] Fix command names in load_save_disk_commands --- lib/ProxySQL_Admin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 1ed7de1733..f5524535a9 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1001,7 +1001,7 @@ static void generate_load_save_disk_commands(std::vector& vec1, std static void generate_load_save_disk_commands(const string& name, const string& command) { std::vector vec1; std::vector vec2; - generate_load_save_disk_commands(vec1, vec2, name); + generate_load_save_disk_commands(vec1, vec2, command); std::tuple, vector> a = tuple, vector>{command, vec1, vec2}; load_save_disk_commands[name] = a; } From 7e382f0a3fdcd567ad430f71e111c4d65d602dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Jan 2023 23:13:11 +0000 Subject: [PATCH 06/13] Avoid duplicate code in vacuum_stats() --- lib/ProxySQL_Admin.cpp | 61 +++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f5524535a9..15d794ed08 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -5224,37 +5224,38 @@ void ProxySQL_Admin::vacuum_stats(bool is_admin) { if (variables.vacuum_stats==false) { return; } - if (is_admin) { - admindb->execute("DELETE FROM stats.stats_mysql_commands_counters"); - admindb->execute("DELETE FROM stats.stats_mysql_free_connections"); - admindb->execute("DELETE FROM stats.stats_mysql_connection_pool"); - admindb->execute("DELETE FROM stats.stats_mysql_connection_pool_reset"); - admindb->execute("DELETE FROM stats.stats_mysql_prepared_statements_info"); - admindb->execute("DELETE FROM stats.stats_mysql_processlist"); - admindb->execute("DELETE FROM stats.stats_mysql_query_digest"); - admindb->execute("DELETE FROM stats.stats_mysql_query_digest_reset"); - admindb->execute("DELETE FROM stats.stats_mysql_query_rules"); - admindb->execute("DELETE FROM stats.stats_mysql_users"); - admindb->execute("DELETE FROM stats.stats_proxysql_servers_checksums"); - admindb->execute("DELETE FROM stats.stats_proxysql_servers_metrics"); - admindb->execute("DELETE FROM stats.stats_proxysql_servers_status"); - admindb->execute("VACUUM stats"); + const vector tablenames = { + "stats_mysql_commands_counters", + "stats_mysql_free_connections", + "stats_mysql_connection_pool", + "stats_mysql_connection_pool_reset", + "stats_mysql_prepared_statements_info", + "stats_mysql_processlist", + "stats_mysql_query_digest", + "stats_mysql_query_digest_reset", + "stats_mysql_query_rules", + "stats_mysql_users", + "stats_proxysql_servers_checksums", + "stats_proxysql_servers_metrics", + "stats_proxysql_servers_status", + }; + string s; + SQLite3DB *tmpdb = NULL; + if (is_admin == true) { + tmpdb = admindb; } else { - statsdb->execute("DELETE FROM stats_mysql_commands_counters"); - statsdb->execute("DELETE FROM stats_mysql_free_connections"); - statsdb->execute("DELETE FROM stats_mysql_connection_pool"); - statsdb->execute("DELETE FROM stats_mysql_connection_pool_reset"); - statsdb->execute("DELETE FROM stats_mysql_prepared_statements_info"); - statsdb->execute("DELETE FROM stats_mysql_processlist"); - statsdb->execute("DELETE FROM stats_mysql_query_digest"); - statsdb->execute("DELETE FROM stats_mysql_query_digest_reset"); - statsdb->execute("DELETE FROM stats_mysql_query_rules"); - statsdb->execute("DELETE FROM stats_mysql_users"); - statsdb->execute("DELETE FROM stats_proxysql_servers_checksums"); - statsdb->execute("DELETE FROM stats_proxysql_servers_metrics"); - statsdb->execute("DELETE FROM stats_proxysql_servers_status"); - statsdb->execute("VACUUM"); - } + tmpdb = statsdb; + } + for (auto it = tablenames.begin(); it != tablenames.end(); it++) { + s = "DELETE FROM "; + if (is_admin == true) s+= "stats."; + s += *it; + tmpdb->execute(s.c_str()); + } + s = "VACUUM"; + if (is_admin == true) + s+= " stats"; + tmpdb->execute(s.c_str()); } From 23a36eea004344ab42d223b74ed8bcbba83b08d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 24 Jan 2023 23:12:09 +0000 Subject: [PATCH 07/13] Fixing 2 typos --- lib/MySQL_HostGroups_Manager.cpp | 2 +- lib/ProxySQL_Admin.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 766e4ef35c..41c9d9fcb2 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3683,7 +3683,7 @@ SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& return this->incoming_group_replication_hostgroups; } else if (name == "mysql_replication_hostgroups") { return this->incoming_replication_hostgroups; - } else if (name == "mysql_serverss") { + } else if (name == "mysql_servers") { return this->runtime_mysql_servers; } else { assert(0); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 15d794ed08..d8909a6fe8 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -10399,7 +10399,7 @@ void ProxySQL_Admin::flush_GENERIC__from_to(const string& name, const string& di if (direction == "disk_to_memory") { BQE1(admindb, it->second, "DELETE FROM main.", "INSERT INTO main.", " SELECT * FROM disk."); } else if (direction == "memory_to_disk") { - BQE1(admindb, it->second, "DELETE FROM disk.", "INSERT INTO disk.", " SELECT * FROM man."); + BQE1(admindb, it->second, "DELETE FROM disk.", "INSERT INTO disk.", " SELECT * FROM main."); } else { assert(0); } From d61c043bdc2182fc20e38a6275654dd3546c0889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 28 Jan 2023 14:07:53 +0000 Subject: [PATCH 08/13] Drafting mysql_hostgroup_attributes table --- include/MySQL_HostGroups_Manager.h | 25 +++++ include/ProxySQL_Cluster.hpp | 7 +- include/proxysql_admin.h | 3 +- lib/MySQL_HostGroups_Manager.cpp | 149 +++++++++++++++++++++++++++++ lib/ProxySQL_Admin.cpp | 91 +++++++++++++++++- lib/ProxySQL_Cluster.cpp | 73 +++++++++++++- 6 files changed, 343 insertions(+), 5 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index bfd6f329ed..e1df6480f1 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -23,6 +23,9 @@ #define SPOOKYV2 #endif +#include "../deps/json/json.hpp" +using json = nlohmann::json; + #ifdef DEBUG /* */ // Enabling STRESSTEST_POOL ProxySQL will do a lot of loops in the connection pool @@ -61,6 +64,9 @@ #define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" +#define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + + typedef std::unordered_map umap_mysql_errors; class MySrvConnList; @@ -202,6 +208,21 @@ class MyHGC { // MySQL Host Group Container unsigned long long current_time_now; uint32_t new_connections_now; MySrvList *mysrvs; + struct { // this is a series of attributes specific for each hostgroup + char * init_connect; + char * comment; + char * ignore_session_variables_text; // this is the original version (text format) of ignore_session_variables + uint32_t max_num_online_servers; + uint32_t throttle_connections_per_sec; + int8_t autocommit; + int8_t free_connections_pct; + bool multiplex; + bool connection_warming; + bool configured; // this variable controls if attributes are configured or not. If not configured, they do not apply + bool initialized; // this variable controls if attributes were ever configured or not. Used by reset_attributes() + json ignore_session_variables_json; // the JSON format of ignore_session_variables + } attributes; + void reset_attributes(); MyHGC(int); ~MyHGC(); MySrvC *get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, MySQL_Session *sess); @@ -409,6 +430,7 @@ class MySQL_HostGroups_Manager { * - 'CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS' * - 'CLUSTER_QUERY_MYSQL_GALERA' * - 'CLUSTER_QUERY_MYSQL_AWS_AURORA' + * - 'CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES' * Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets. */ SQLite3_result *incoming_replication_hostgroups; @@ -431,6 +453,9 @@ class MySQL_HostGroups_Manager { pthread_mutex_t AWS_Aurora_Info_mutex; std::map AWS_Aurora_Info_Map; + void generate_mysql_hostgroup_attributes_table(); + SQLite3_result *incoming_hostgroup_attributes; + std::thread *HGCU_thread; std::thread *GTID_syncer_thread; diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 45a319b62a..7be5700503 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -20,7 +20,7 @@ * also represent the actual resultset being received when issuing them, since this resultset is used for computing the * 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules: * - 'runtime_mysql_servers': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups', - * 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups'. + * 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'. * - 'runtime_mysql_users'. * - 'runtime_mysql_query_rules'. * @@ -37,6 +37,9 @@ /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_group_replication_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup" +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_hostgroup_attributes'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES "PROXY_SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment FROM runtime_mysql_hostgroup_attributes ORDER BY hostgroup_id" + /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" @@ -299,6 +302,8 @@ struct p_cluster_counter { pulled_mysql_servers_galera_hostgroups_failure, pulled_mysql_servers_aws_aurora_hostgroups_success, pulled_mysql_servers_aws_aurora_hostgroups_failure, + pulled_mysql_servers_hostgroup_attributes_success, + pulled_mysql_servers_hostgroup_attributes_failure, pulled_mysql_servers_runtime_checks_success, pulled_mysql_servers_runtime_checks_failure, diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 149b69cf49..ce15efa67c 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -122,9 +122,10 @@ struct incoming_servers_t { SQLite3_result* incoming_group_replication_hostgroups = NULL; SQLite3_result* incoming_galera_hostgroups = NULL; SQLite3_result* incoming_aurora_hostgroups = NULL; + SQLite3_result* incoming_hostgroup_attributes = NULL; incoming_servers_t(); - incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); + incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); }; class ProxySQL_Admin { diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 41c9d9fcb2..c1c0de12aa 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -884,8 +884,29 @@ MyHGC::MyHGC(int _hid) { mysrvs=new MySrvList(this); current_time_now = 0; new_connections_now = 0; + attributes.initialized = false; + reset_attributes(); } +void MyHGC::reset_attributes() { + if (attributes.initialized == false) { + attributes.init_connect = NULL; + attributes.comment = NULL; + attributes.ignore_session_variables_text = NULL; + } + attributes.initialized = true; + attributes.configured = false; + attributes.max_num_online_servers = 1000000; + attributes.throttle_connections_per_sec = 1000000; + attributes.autocommit = -1; + attributes.free_connections_pct = 10; + attributes.multiplex = true; + attributes.connection_warming = false; + free(attributes.init_connect); + free(attributes.comment); + free(attributes.ignore_session_variables_text); + attributes.ignore_session_variables_json = json(); +} MyHGC::~MyHGC() { delete mysrvs; @@ -1322,6 +1343,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { mydb->execute(MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_GALERA_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES); mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)"); MyHostGroups=new PtrArray(); runtime_mysql_servers=NULL; @@ -1329,6 +1351,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { incoming_group_replication_hostgroups=NULL; incoming_galera_hostgroups=NULL; incoming_aws_aurora_hostgroups = NULL; + incoming_hostgroup_attributes = NULL; pthread_rwlock_init(>id_rwlock, NULL); gtid_missing_nodes = false; gtid_ev_loop=NULL; @@ -1585,6 +1608,7 @@ void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& m CUCFT1(myhash,init,"mysql_group_replication_hostgroups","writer_hostgroup"); CUCFT1(myhash,init,"mysql_galera_hostgroups","writer_hostgroup"); CUCFT1(myhash,init,"mysql_aws_aurora_hostgroups","writer_hostgroup"); + CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id"); } bool MySQL_HostGroups_Manager::commit( @@ -1829,6 +1853,13 @@ bool MySQL_HostGroups_Manager::commit( generate_mysql_aws_aurora_hostgroups_table(); } + // hostgroup attributes + if (incoming_hostgroup_attributes) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_hostgroup_attributes\n"); + mydb->execute("DELETE FROM mysql_hostgroup_attributes"); + generate_mysql_hostgroup_attributes_table(); + } + if ( GloAdmin && GloAdmin->checksum_variables.checksum_mysql_servers ) { uint64_t hash1=0, hash2=0; @@ -2474,6 +2505,8 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_group_replication_hostgroups"; } else if (name == "mysql_replication_hostgroups") { query=(char *)"SELECT writer_hostgroup, reader_hostgroup, check_type, comment FROM mysql_replication_hostgroups"; + } else if (name == "mysql_hostgroup_attributes") { + query=(char *)"SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; } else if (name == "mysql_servers") { query = (char *)MYHGM_GEN_ADMIN_RUNTIME_SERVERS; } else { @@ -3656,6 +3689,8 @@ void MySQL_HostGroups_Manager::save_incoming_mysql_table(SQLite3_result *s, cons inc = &incoming_group_replication_hostgroups; } else if (name == "mysql_replication_hostgroups") { inc = &incoming_replication_hostgroups; + } else if (name == "mysql_hostgroup_attributes") { + inc = &incoming_hostgroup_attributes; } else { assert(0); } @@ -3683,6 +3718,8 @@ SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& return this->incoming_group_replication_hostgroups; } else if (name == "mysql_replication_hostgroups") { return this->incoming_replication_hostgroups; + } else if (name == "mysql_hostgroup_attributes") { + return this->incoming_hostgroup_attributes; } else if (name == "mysql_servers") { return this->runtime_mysql_servers; } else { @@ -6406,6 +6443,118 @@ bool AWS_Aurora_Info::update(int r, int _port, char *_end_addr, int maxl, int al return ret; } +void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() { +// TODO TODO TODO + if (incoming_hostgroup_attributes==NULL) { + return; + } + int rc; + sqlite3_stmt *statement=NULL; + + const char * query=(const char *)"INSERT INTO mysql_hostgroup_attributes ( " + "hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, " + "init_connect, multiplex, connection_warming, throttle_connections_per_sec, " + "ignore_session_variables, comment) VALUES " + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8, ?10)"; + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query, -1, &statement, 0); + rc = mydb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, mydb); + proxy_info("New mysql_hostgroup_attributes table\n"); + // set configured = false to all + // in this way later we can known which HG were updated + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + myhgc->attributes.configured = false; + } + + /** + * @brief We iterate the whole resultset incoming_hostgroup_attributes and configure + * both the hostgroup in memory, but also pupulate table mysql_hostgroup_attributes + * connection errors. + * @details for each row in incoming_hostgroup_attributes: + * 1. it finds (or create) the hostgroup + * 2. it writes the in mysql_hostgroup_attributes + * 3. it finds (or create) the attributes of the hostgroup + */ + for (std::vector::iterator it = incoming_hostgroup_attributes->rows.begin() ; it != incoming_hostgroup_attributes->rows.end(); ++it) { + SQLite3_row *r=*it; + unsigned int hid = (unsigned int)atoi(r->fields[0]); + MyHGC *myhgc = MyHGC_lookup(hid); // note: MyHGC_lookup() will create the HG if doesn't exist! + int max_num_online_servers = atoi(r->fields[1]); + int autocommit = atoi(r->fields[2]); + int free_connections_pct = atoi(r->fields[3]); + char * init_connect = r->fields[4]; + int multiplex = atoi(r->fields[5]); + int connection_warming = atoi(r->fields[6]); + int throttle_connections_per_sec = atoi(r->fields[7]); + char * ignore_session_variables = r->fields[8]; + char * comment = r->fields[9]; + proxy_info("Loading MySQL Hostgroup Attributes info for (%d,%d,%d,%d,\"%s\",%d,%d,%d,\"%s\",\"%s\")\n", + hid, max_num_online_servers, autocommit, free_connections_pct, + init_connect, multiplex, connection_warming, throttle_connections_per_sec, + ignore_session_variables, comment + ); + rc=(*proxy_sqlite3_bind_int64)(statement, 1, hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, max_num_online_servers); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 3, autocommit); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4, free_connections_pct); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 5, init_connect, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, multiplex); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7, connection_warming); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, throttle_connections_per_sec); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 9, ignore_session_variables, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 10, comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb); + myhgc->attributes.configured = true; + myhgc->attributes.max_num_online_servers = max_num_online_servers; + myhgc->attributes.autocommit = autocommit; + myhgc->attributes.free_connections_pct = free_connections_pct; + myhgc->attributes.multiplex = multiplex; + myhgc->attributes.connection_warming = connection_warming; + myhgc->attributes.throttle_connections_per_sec = throttle_connections_per_sec; + if (myhgc->attributes.init_connect != NULL) + free(myhgc->attributes.init_connect); + myhgc->attributes.init_connect = strdup(init_connect); + if (myhgc->attributes.comment != NULL) + free(myhgc->attributes.comment); + myhgc->attributes.init_connect = strdup(comment); + // for ignore_session_variables we store 2 versions: + // 1. the text + // 2. the JSON + // Because calling JSON functions is expensive, we first verify if it changes + if (myhgc->attributes.ignore_session_variables_text == NULL) { + myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); + if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON + myhgc->attributes.ignore_session_variables_json = json::parse(ignore_session_variables); + } + } else { + if (strcmp(myhgc->attributes.ignore_session_variables_text, ignore_session_variables) != 0) { + free(myhgc->attributes.ignore_session_variables_text); + myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); + if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON + myhgc->attributes.ignore_session_variables_json = json::parse(ignore_session_variables); + } + // TODO: assign the variables + } + } + break; + } + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + if (myhgc->attributes.configured == false) { + // if configured == false reset to defaults + proxy_info("Resetting hostgroup attributes for hostgroup %u\n", myhgc->hid); + myhgc->reset_attributes(); + } + } + + delete incoming_hostgroup_attributes; + incoming_hostgroup_attributes=NULL; +} + void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (incoming_aws_aurora_hostgroups==NULL) { return; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index d8909a6fe8..e0aef211c1 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -105,6 +105,7 @@ static const vector mysql_servers_tablenames = { "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", "mysql_aws_aurora_hostgroups", + "mysql_hostgroup_attributes", }; static const vector mysql_firewall_tablenames = { @@ -583,6 +584,11 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR , UNIQUE (reader_hostgroup))" +#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE runtime_mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" // Cluster solution @@ -1054,13 +1060,15 @@ incoming_servers_t::incoming_servers_t( SQLite3_result* incoming_replication_hostgroups, SQLite3_result* incoming_group_replication_hostgroups, SQLite3_result* incoming_galera_hostgroups, - SQLite3_result* incoming_aurora_hostgroups + SQLite3_result* incoming_aurora_hostgroups, + SQLite3_result* incoming_hostgroup_attributes ) : runtime_mysql_servers(runtime_mysql_servers), incoming_replication_hostgroups(incoming_replication_hostgroups), incoming_group_replication_hostgroups(incoming_group_replication_hostgroups), incoming_galera_hostgroups(incoming_galera_hostgroups), - incoming_aurora_hostgroups(incoming_aurora_hostgroups) + incoming_aurora_hostgroups(incoming_aurora_hostgroups), + incoming_hostgroup_attributes(incoming_hostgroup_attributes) {} int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { @@ -3183,6 +3191,8 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign strstr(query_no_space,"runtime_mysql_galera_hostgroups") || strstr(query_no_space,"runtime_mysql_aws_aurora_hostgroups") + || + strstr(query_no_space,"runtime_mysql_hostgroup_attributes") ) { runtime_mysql_servers=true; refresh=true; } @@ -3688,6 +3698,8 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { tn = "mysql_galera_hostgroups"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_AURORA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_AURORA))) { tn = "mysql_aws_aurora_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES, query_no_space, strlen(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES))) { + tn = "mysql_hostgroup_attributes"; } if (tn != "") { GloAdmin->mysql_servers_wrlock(); @@ -4785,6 +4797,15 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { tablename=(char *)"MYSQL AURORA HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; + tablename=(char *)"MYSQL HOSTGROUP ATTRIBUTES"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } if (error) { proxy_error("Error: %s\n", error); @@ -5950,6 +5971,8 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_admin,"runtime_mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_admin,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); insert_into_tables_defs(tables_defs_admin,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_query_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES); @@ -5985,6 +6008,7 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_config,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); insert_into_tables_defs(tables_defs_config,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); insert_into_tables_defs(tables_defs_config,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); @@ -11483,6 +11507,7 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { // make sure that the caller has called mysql_servers_wrlock() char *query=NULL; + string StrQuery; SQLite3_result *resultset=NULL; // dump mysql_servers if (_runtime) { @@ -11740,6 +11765,47 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { } if(resultset) delete resultset; resultset=NULL; + + // dump mysql_hostgroup_attributes + + StrQuery = "DELETE FROM main."; + if (_runtime) + StrQuery += "runtime_"; + StrQuery += "mysql_hostgroup_attributes"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", StrQuery.c_str()); + admindb->execute(StrQuery.c_str()); + resultset=MyHGM->dump_table_mysql("mysql_hostgroup_attributes"); + if (resultset) { + int rc; + sqlite3_stmt *statement=NULL; + StrQuery = "INSERT INTO "; + if (_runtime) + StrQuery += "runtime_"; + StrQuery += "mysql_hostgroup_attributes (hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8, ?10)"; + rc = admindb->prepare_v2(StrQuery.c_str(), &statement); + ASSERT_SQLITE_OK(rc, admindb); + //proxy_info("New mysql_aws_aurora_hostgroups table\n"); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + rc=(*proxy_sqlite3_bind_int64)(statement, 1, atol(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); // hostgroup_id + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atol(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); // max_num_online_servers + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atol(r->fields[2])); ASSERT_SQLITE_OK(rc, admindb); // autocommit + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atol(r->fields[3])); ASSERT_SQLITE_OK(rc, admindb); // free_connections_pct + rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // variable_name + rc=(*proxy_sqlite3_bind_int64)(statement, 6, atol(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); // multiplex + rc=(*proxy_sqlite3_bind_int64)(statement, 7, atol(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); // connection_warming + rc=(*proxy_sqlite3_bind_int64)(statement, 8, atol(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); // throttle_connections_per_sec + rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // ignore_session_variables + rc=(*proxy_sqlite3_bind_text)(statement, 10, r->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // comment + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, admindb); + } + (*proxy_sqlite3_finalize)(statement); + } + if(resultset) delete resultset; + resultset=NULL; } @@ -11772,12 +11838,14 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( SQLite3_result *resultset_group_replication=NULL; SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; + SQLite3_result *resultset_hostgroup_attributes=NULL; SQLite3_result* runtime_mysql_servers = incoming_servers.runtime_mysql_servers; SQLite3_result* incoming_replication_hostgroups = incoming_servers.incoming_replication_hostgroups; SQLite3_result* incoming_group_replication_hostgroups = incoming_servers.incoming_group_replication_hostgroups; SQLite3_result* incoming_galera_hostgroups = incoming_servers.incoming_galera_hostgroups; SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups; + SQLite3_result* incoming_hostgroup_attributes = incoming_servers.incoming_hostgroup_attributes; char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); @@ -11922,6 +11990,22 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( // Pass the resultset to MyHGM MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); } + + // support for hostgroup attributes, table mysql_hostgroup_attributes + query = (char *)"SELECT * FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + if (incoming_hostgroup_attributes == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_hostgroup_attributes); + } else { + resultset_hostgroup_attributes = incoming_hostgroup_attributes; + } + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + // Pass the resultset to MyHGM + MyHGM->save_incoming_mysql_table(resultset_hostgroup_attributes, "mysql_hostgroup_attributes"); + } + // commit all the changes MyHGM->commit(runtime_mysql_servers, checksum, epoch); GloAdmin->save_mysql_servers_runtime_to_database(true); @@ -11945,6 +12029,9 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( //delete resultset_aws_aurora; // do not delete, resultset is stored in MyHGM resultset_aws_aurora=NULL; } + if (resultset_hostgroup_attributes) { + resultset_hostgroup_attributes = NULL; + } } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index f6dda66549..695a6415a8 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1431,6 +1431,7 @@ uint64_t mysql_servers_raw_checksum(MYSQL_RES* resultset) { * - CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS. * - CLUSTER_QUERY_MYSQL_GALERA. * - CLUSTER_QUERY_MYSQL_AWS_AURORA. + * - CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES. * * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in * the specified order. @@ -1477,7 +1478,8 @@ incoming_servers_t convert_servers_resultsets(const std::vector& res get_SQLite3_resulset(results[1]).release(), get_SQLite3_resulset(results[2]).release(), get_SQLite3_resulset(results[3]).release(), - get_SQLite3_resulset(results[4]).release() + get_SQLite3_resulset(results[4]).release(), + get_SQLite3_resulset(results[5]).release() }; } } @@ -1538,6 +1540,12 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, std::string fetch_galera_err = ""; string_format("Cluster: Fetching 'MySQL Galera Hostgroups' from peer %s:%d failed: \n", fetch_galera_err, hostname, port); + // hostgroup attributes messages + std::string fetch_hostgroup_attributes_start = ""; + string_format("Cluster: Fetching 'MySQL Hostgroup Attributes' from peer %s:%d\n", fetch_hostgroup_attributes_start, hostname, port); + std::string fetch_hostgroup_attributes_err = ""; + string_format("Cluster: Fetching 'MySQL Hostgroup Attributes' from peer %s:%d failed: \n", fetch_hostgroup_attributes_err, hostname, port); + // Create fetching queries /** @@ -1574,6 +1582,12 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, p_cluster_counter::pulled_mysql_servers_aws_aurora_hostgroups_success, p_cluster_counter::pulled_mysql_servers_aws_aurora_hostgroups_failure, { fetch_aws_aurora_start, "", fetch_aws_aurora_err } + }, + { + CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES, + p_cluster_counter::pulled_mysql_servers_hostgroup_attributes_success, + p_cluster_counter::pulled_mysql_servers_hostgroup_attributes_failure, + { fetch_hostgroup_attributes_start, "", fetch_hostgroup_attributes_err } } }; @@ -1770,6 +1784,42 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, resultset->dump_to_stderr(); delete resultset; + // sync mysql_hostgroup_attributes + proxy_info("Cluster: Writing mysql_hostgroup_attributes table\n"); + GloAdmin->admindb->execute("DELETE FROM mysql_hostgroup_attributes"); + { + const char * q=(const char *)"INSERT INTO mysql_hostgroup_attributes ( " + "hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, " + "init_connect, multiplex, connection_warming, throttle_connections_per_sec, " + "ignore_session_variables, comment) VALUES " + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8, ?10)"; + sqlite3_stmt *statement1 = NULL; + int rc = GloAdmin->admindb->prepare_v2(q, &statement1); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + + while ((row = mysql_fetch_row(results[5]))) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // hostgroup_id + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_num_online_servers + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atol(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // autocommit + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atol(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // free_connections_pct + rc=(*proxy_sqlite3_bind_text)(statement1, 5, row[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // variable_name + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // multiplex + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atol(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // connection_warming + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // throttle_connections_per_sec + rc=(*proxy_sqlite3_bind_text)(statement1, 9, row[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // ignore_session_variables + rc=(*proxy_sqlite3_bind_text)(statement1, 10, row[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + (*proxy_sqlite3_finalize)(statement1); + } + + proxy_info("Dumping fetched 'mysql_hostgroup_attributes'\n"); + GloAdmin->admindb->execute_statement((char *)"SELECT * FROM mysql_hostgroup_attributes", &error , &cols , &affected_rows , &resultset); + resultset->dump_to_stderr(); + delete resultset; + proxy_info("Cluster: Loading to runtime MySQL Servers from peer %s:%d\n", hostname, port); GloAdmin->load_mysql_servers_to_runtime(incoming_servers, checksum, epoch); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { @@ -3341,6 +3391,27 @@ cluster_metrics_map = std::make_tuple( ), // ==================================================================== + // ==================================================================== + std::make_tuple ( + p_cluster_counter::pulled_mysql_servers_hostgroup_attributes_success, + "proxysql_cluster_pulled_total", + "Number of times a 'module' have been pulled from a peer.", + metric_tags { + { "module_name", "mysql_servers_hostgroup_attributes" }, + { "status", "success" } + } + ), + std::make_tuple ( + p_cluster_counter::pulled_mysql_servers_hostgroup_attributes_failure, + "proxysql_cluster_pulled_total", + "Number of times a 'module' have been pulled from a peer.", + metric_tags { + { "module_name", "mysql_servers_hostgroup_attributes" }, + { "status", "failure" } + } + ), + // ==================================================================== + // ==================================================================== std::make_tuple ( p_cluster_counter::pulled_mysql_servers_runtime_checks_success, From 9235fbf887f66e5fc6b9da29a2b2fc407b95bc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 29 Jan 2023 01:14:44 +0000 Subject: [PATCH 09/13] Added 1st testing for mysql_hostgroup_attributes This commit adds some minimal testing for mysql_hostgroup_attributes . It helped detecting some bugs. --- lib/MySQL_HostGroups_Manager.cpp | 18 ++- lib/ProxySQL_Admin.cpp | 12 +- lib/ProxySQL_Cluster.cpp | 2 +- test/cluster/check_all_nodes.bash | 2 +- test/tap/tests/admin_various_commands2-t.cpp | 3 + .../test_mysql_hostgroup_attributes-1-t.cpp | 103 ++++++++++++++++++ 6 files changed, 131 insertions(+), 9 deletions(-) create mode 100644 test/tap/tests/test_mysql_hostgroup_attributes-1-t.cpp diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c1c0de12aa..e42b2bfb0e 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -903,12 +903,16 @@ void MyHGC::reset_attributes() { attributes.multiplex = true; attributes.connection_warming = false; free(attributes.init_connect); + attributes.init_connect = NULL; free(attributes.comment); + attributes.comment = NULL; free(attributes.ignore_session_variables_text); + attributes.ignore_session_variables_text = NULL; attributes.ignore_session_variables_json = json(); } MyHGC::~MyHGC() { + reset_attributes(); // free all memory delete mysrvs; } @@ -1597,9 +1601,11 @@ void MySQL_HostGroups_Manager::CUCFT1(SpookyHash& myhash, bool& init, const stri } uint64_t hash1_ = resultset->raw_checksum(); myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is %lu\n", TableName.c_str(), hash1_); + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), hash1_); } delete resultset; + } else { + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), (long unsigned int)0); } } @@ -1914,9 +1920,11 @@ bool MySQL_HostGroups_Manager::commit( } uint64_t hash1_ = resultset->raw_checksum(); myhash.Update(&hash1_, sizeof(hash1_)); - proxy_info("Checksum for table %s is %lu\n", "mysql_servers", hash1_); + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", hash1_); } delete resultset; + } else { + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); } } @@ -6444,7 +6452,6 @@ bool AWS_Aurora_Info::update(int r, int _port, char *_end_addr, int maxl, int al } void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() { -// TODO TODO TODO if (incoming_hostgroup_attributes==NULL) { return; } @@ -6455,7 +6462,7 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() { "hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, " "init_connect, multiplex, connection_warming, throttle_connections_per_sec, " "ignore_session_variables, comment) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8, ?10)"; + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query, -1, &statement, 0); rc = mydb->prepare_v2(query, &statement); @@ -6520,7 +6527,7 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() { myhgc->attributes.init_connect = strdup(init_connect); if (myhgc->attributes.comment != NULL) free(myhgc->attributes.comment); - myhgc->attributes.init_connect = strdup(comment); + myhgc->attributes.comment = strdup(comment); // for ignore_session_variables we store 2 versions: // 1. the text // 2. the JSON @@ -6540,7 +6547,6 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() { // TODO: assign the variables } } - break; } for (unsigned int i=0; ilen; i++) { MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index e0aef211c1..0da211bb5d 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -4814,11 +4814,14 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { SPA->send_MySQL_ERR(&sess->client_myds->myprot, buf); run_query=false; } else if (resultset) { + l_free(query_length,query); char *q=(char *)"SELECT '%s' AS 'table', '%s' AS 'checksum'"; char *checksum=(char *)resultset->checksum(); query=(char *)malloc(strlen(q)+strlen(tablename)+strlen(checksum)+1); sprintf(query,q,tablename,checksum); + query_length = strlen(query); free(checksum); + delete resultset; } goto __run_query; } @@ -11781,7 +11784,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { StrQuery = "INSERT INTO "; if (_runtime) StrQuery += "runtime_"; - StrQuery += "mysql_hostgroup_attributes (hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8, ?10)"; + StrQuery += "mysql_hostgroup_attributes (hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; rc = admindb->prepare_v2(StrQuery.c_str(), &statement); ASSERT_SQLITE_OK(rc, admindb); //proxy_info("New mysql_aws_aurora_hostgroups table\n"); @@ -11860,6 +11863,13 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( } else { MyHGM->servers_add(resultset_servers); } + // memory leak detected here + if (runtime_mysql_servers == nullptr) { + if (resultset_servers != nullptr) { + delete resultset_servers; + resultset_servers = nullptr; + } + } resultset=NULL; query=(char *)"SELECT a.* FROM mysql_replication_hostgroups a JOIN mysql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup"; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 695a6415a8..c70c3b8959 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1792,7 +1792,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, "hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, " "init_connect, multiplex, connection_warming, throttle_connections_per_sec, " "ignore_session_variables, comment) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8, ?10)"; + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; sqlite3_stmt *statement1 = NULL; int rc = GloAdmin->admindb->prepare_v2(q, &statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); diff --git a/test/cluster/check_all_nodes.bash b/test/cluster/check_all_nodes.bash index afae05d1e4..1953e252e5 100644 --- a/test/cluster/check_all_nodes.bash +++ b/test/cluster/check_all_nodes.bash @@ -1,5 +1,5 @@ #!/bin/bash -TABLES=(mysql_servers mysql_users mysql_query_rules mysql_query_rules_fast_routing global_variables proxysql_servers mysql_galera_hostgroups mysql_group_replication_hostgroups mysql_replication_hostgroups) +TABLES=(mysql_servers mysql_users mysql_query_rules mysql_query_rules_fast_routing global_variables proxysql_servers mysql_galera_hostgroups mysql_group_replication_hostgroups mysql_replication_hostgroups mysql_hostgroup_attributes) ALL_TABLES=() diff --git a/test/tap/tests/admin_various_commands2-t.cpp b/test/tap/tests/admin_various_commands2-t.cpp index 3f3b73da73..d38c2cc409 100644 --- a/test/tap/tests/admin_various_commands2-t.cpp +++ b/test/tap/tests/admin_various_commands2-t.cpp @@ -57,6 +57,7 @@ std::vector queries = { "CHECKSUM MEM MYSQL QUERY RULES", "CHECKSUM MEM MYSQL VARIABLES", "CHECKSUM MEM MYSQL REPLICATION HOSTGROUPS", + "CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES", "CHECKSUM MEM MYSQL GROUP REPLICATION HOSTGROUPS", "CHECKSUM MEM MYSQL GALERA HOSTGROUPS", "CHECKSUM MEM MYSQL AURORA HOSTGROUPS", @@ -65,6 +66,7 @@ std::vector queries = { "CHECKSUM MEMORY MYSQL QUERY RULES", "CHECKSUM MEMORY MYSQL VARIABLES", "CHECKSUM MEMORY MYSQL REPLICATION HOSTGROUPS", + "CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES", "CHECKSUM MEMORY MYSQL GROUP REPLICATION HOSTGROUPS", "CHECKSUM MEMORY MYSQL GALERA HOSTGROUPS", "CHECKSUM MEMORY MYSQL AURORA HOSTGROUPS", @@ -73,6 +75,7 @@ std::vector queries = { "CHECKSUM MYSQL QUERY RULES", "CHECKSUM MYSQL VARIABLES", "CHECKSUM MYSQL REPLICATION HOSTGROUPS", + "CHECKSUM MYSQL HOSTGROUP ATTRIBUTES", "CHECKSUM MYSQL GROUP REPLICATION HOSTGROUPS", "CHECKSUM MYSQL GALERA HOSTGROUPS", "CHECKSUM MYSQL AURORA HOSTGROUPS", diff --git a/test/tap/tests/test_mysql_hostgroup_attributes-1-t.cpp b/test/tap/tests/test_mysql_hostgroup_attributes-1-t.cpp new file mode 100644 index 0000000000..f37be3912b --- /dev/null +++ b/test/tap/tests/test_mysql_hostgroup_attributes-1-t.cpp @@ -0,0 +1,103 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + + +int run_queries_sets(std::vector& queries, MYSQL *my, const std::string& message_prefix) { + for (std::vector::iterator it = queries.begin(); it != queries.end(); it++) { + std::string q = *it; + diag("%s: %s", message_prefix.c_str(), q.c_str()); + MYSQL_QUERY(my, q.c_str()); + } + return 0; +} + +int check_checksum(MYSQL *mysqladmin, const char *expected_checksum, const char *when) { + MYSQL_QUERY(mysqladmin, "CHECKSUM MYSQL HOSTGROUP ATTRIBUTES"); + MYSQL_RES* result = mysql_store_result(mysqladmin); + ok(mysql_num_rows(result) == 1, "Line %d: CHECKSUM returned 1 row" , __LINE__); + if (mysql_num_rows(result) == 1) { + MYSQL_ROW row = mysql_fetch_row(result); + ok(strcmp(row[1],expected_checksum)==0, "Checksum %s: expected: %s, returned: %s", when, expected_checksum, row[1]); + } + mysql_free_result(result); + return 0; +} + +int run_one_test(MYSQL *mysqladmin, const char *expected_checksum, const char *query) { + std::vector queries = { "DELETE FROM mysql_hostgroup_attributes" }; + queries.push_back(std::string(query)); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + if (run_queries_sets(queries, mysqladmin, "Running on Admin")) + return 1; + if (check_checksum(mysqladmin,expected_checksum,"before")) + return 1; + queries = { "DELETE FROM mysql_hostgroup_attributes", "SAVE MYSQL SERVERS FROM RUNTIME" }; + if (run_queries_sets(queries, mysqladmin, "Running on Admin")) + return 1; + if (check_checksum(mysqladmin,expected_checksum,"after")) + return 1; + return 0; +} + +int main(int argc, char** argv) { + CommandLine cl; + + if(cl.getEnv()) + return exit_status(); + + + std::unordered_map queries_and_checksums = { + { + "0x666CFBEEDB76EE9C", + "INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'','')" + }, + { + "0xE2FC2A5FEE8D18DC", + "INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'',''),(18,2,-1,20,'SET sql_mode=\"\"',0,0,100,'','hello world')", + }, + { + "0xFACE1C64FF1C373E", + "INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'',''),(18,2,-1,20,'SET sql_mode=\"\"',0,0,100,'','hello world'),(17,0,0,30,'SET long_query_time=0',1,0,123,'{\"session_variables\":[\"tmp_table_size\",\"join_buffer_size\"]}','filtering variables')" + }, + }; + + plan(queries_and_checksums.size()*4); + diag("Testing the loading of mysql_hostgroup_attributes"); + + MYSQL* mysqladmin = mysql_init(NULL); + if (!mysqladmin) + return exit_status(); + + if (!mysql_real_connect(mysqladmin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", + __FILE__, __LINE__, mysql_error(mysqladmin)); + return exit_status(); + } + + for (std::unordered_map::iterator it = queries_and_checksums.begin(); it != queries_and_checksums.end(); it++) { + if (run_one_test(mysqladmin, it->first.c_str() , it->second.c_str()) == 1) + return exit_status(); + auto it2 = it; + it2++; + if (it2 != queries_and_checksums.end()) { + diag("Sleeping 10 seconds because of Cluster"); + sleep(10); + } + } + mysql_close(mysqladmin); + + return exit_status(); +} + From 3837de61e294a3de5a7b88b638fba1dd14ae10d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 29 Jan 2023 18:09:05 +0000 Subject: [PATCH 10/13] Various minor fixes Fixes: * minor memory leak when running CHECKSUM command * minor memory leak in load_mysql_servers_to_runtime() * not initialized json in MySQL_Event (valgrind doesn't like it) * not initialized variable MySQL_Thread::shutdown in the constructor --- lib/MySQL_Logger.cpp | 4 ++-- lib/MySQL_Thread.cpp | 1 + lib/ProxySQL_Admin.cpp | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/MySQL_Logger.cpp b/lib/MySQL_Logger.cpp index 61fba5f680..9e61c68cc4 100644 --- a/lib/MySQL_Logger.cpp +++ b/lib/MySQL_Logger.cpp @@ -124,7 +124,7 @@ uint64_t MySQL_Event::write(std::fstream *f, MySQL_Session *sess) { } void MySQL_Event::write_auth(std::fstream *f, MySQL_Session *sess) { - json j; + json j = {}; j["timestamp"] = start_time/1000; { time_t timer=start_time/1000/1000; @@ -349,7 +349,7 @@ uint64_t MySQL_Event::write_query_format_1(std::fstream *f) { } uint64_t MySQL_Event::write_query_format_2_json(std::fstream *f) { - json j; + json j = {}; uint64_t total_bytes=0; if (hid!=UINT64_MAX) { j["hostgroup_id"] = hid; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index ba6ed9a071..279adb14d2 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -4200,6 +4200,7 @@ MySQL_Thread::MySQL_Thread() { for (int i=0; iservers_add(resultset_servers); } - // memory leak detected here - if (runtime_mysql_servers == nullptr) { + // memory leak was detected here. The following few lines fix that + if (runtime_mysql_servers == nullptr) { if (resultset_servers != nullptr) { delete resultset_servers; resultset_servers = nullptr; From f3c604eb848d587f2bd9015f436db200799e155b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 29 Jan 2023 21:55:59 +0000 Subject: [PATCH 11/13] Implemented mysql_hostgroup_attributes.multiplex If mysql_hostgroup_attributes.multiplex: - 0 : multiplexing is always disabled - 1 : multiplexing is enabled for the hostgroup, but other criteria can disable multiplexing (transaction, temp tables, query rules, etc) --- include/mysql_connection.h | 2 +- lib/MySQL_HostGroups_Manager.cpp | 6 ++++++ lib/MySQL_Session.cpp | 1 + lib/mysql_connection.cpp | 5 ++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 43e8dadf31..dda947f2da 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -17,7 +17,7 @@ using json = nlohmann::json; #define STATUS_MYSQL_CONNECTION_NO_MULTIPLEX 0x00000080 #define STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0 0x00000100 #define STATUS_MYSQL_CONNECTION_FOUND_ROWS 0x00000200 -//#define STATUS_MYSQL_CONNECTION_NO_BACKSLASH_ESCAPES 0x00000400 +#define STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG 0x00000400 #define STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT 0x00000800 class Variable { diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index e42b2bfb0e..a19056fc58 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3114,6 +3114,8 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff // we must create a new connection conn = new MySQL_Connection(); conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); } @@ -3126,6 +3128,8 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff if ((conns_used > conns_free) && (mysrvc->max_connections > (conns_free/2 + conns_used/2)) ) { conn = new MySQL_Connection(); conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); } else { @@ -3164,6 +3168,8 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff } else { conn = new MySQL_Connection(); conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); return conn; diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index c511f68a56..629d548373 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -1214,6 +1214,7 @@ void MySQL_Session::generate_proxysql_internal_session_json(json &j) { j["backends"][i]["conn"]["status"]["user_variable"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_USER_VARIABLE); j["backends"][i]["conn"]["status"]["found_rows"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_FOUND_ROWS); j["backends"][i]["conn"]["status"]["no_multiplex"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX); + j["backends"][i]["conn"]["status"]["no_multiplex_HG"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); j["backends"][i]["conn"]["status"]["compression"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_COMPRESSION); j["backends"][i]["conn"]["status"]["prepared_statement"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT); j["backends"][i]["conn"]["MultiplexDisabled"] = _myconn->MultiplexDisabled(); diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 7b9944dc0b..fcd06e045b 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -2428,7 +2428,7 @@ bool MySQL_Connection::MultiplexDisabled(bool check_delay_token) { // status_flags stores information about the status of the connection // can be used to determine if multiplexing can be enabled or not bool ret=false; - if (status_flags & (STATUS_MYSQL_CONNECTION_TRANSACTION|STATUS_MYSQL_CONNECTION_USER_VARIABLE|STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT|STATUS_MYSQL_CONNECTION_LOCK_TABLES|STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE|STATUS_MYSQL_CONNECTION_GET_LOCK|STATUS_MYSQL_CONNECTION_NO_MULTIPLEX|STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0|STATUS_MYSQL_CONNECTION_FOUND_ROWS|STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT) ) { + if (status_flags & (STATUS_MYSQL_CONNECTION_TRANSACTION|STATUS_MYSQL_CONNECTION_USER_VARIABLE|STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT|STATUS_MYSQL_CONNECTION_LOCK_TABLES|STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE|STATUS_MYSQL_CONNECTION_GET_LOCK|STATUS_MYSQL_CONNECTION_NO_MULTIPLEX|STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0|STATUS_MYSQL_CONNECTION_FOUND_ROWS|STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG|STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT) ) { ret=true; } if (check_delay_token && auto_increment_delay_token) return true; @@ -2768,7 +2768,10 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne } void MySQL_Connection::reset() { + bool old_no_multiplex_hg = get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); status_flags=0; + // reconfigure STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG + set_status(old_no_multiplex_hg,STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); reusable=true; options.last_set_autocommit=-1; // never sent From 7b00d05233ec8d98b9f4e7053f6d0156b635990c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 29 Jan 2023 22:17:01 +0000 Subject: [PATCH 12/13] mysql_hostgroup_attributes free_connections_pct and connection_warming Implementation of mysql_hostgroup_attributes.free_connections_pct and mysql_hostgroup_attributes.connection_warming If mysql_hostgroup_attributes is configured: - mysql_hostgroup_attributes.free_connections_pct has higher priority than global variable mysql-free_connections_pct - mysql_hostgroup_attributes.connection_warming has higher priority than global variable mysql-connection_warming --- lib/MySQL_HostGroups_Manager.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index a19056fc58..703254f51f 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3061,9 +3061,16 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff // 2 : tracked options are OK , CHANGE USER is not required, but some SET statement or INIT_DB needs to be executed // 3 : tracked options are OK , CHANGE USER is not required, and it seems that SET statements or INIT_DB ARE not required unsigned int number_of_matching_session_variables = 0; // this includes session variables AND schema - if (mysql_thread___connection_warming) { + bool connection_warming = mysql_thread___connection_warming; + int free_connections_pct = mysql_thread___free_connections_pct; + if (mysrvc->myhgc->attributes.configured == true) { + // mysql_hostgroup_attributes takes priority + connection_warming = mysrvc->myhgc->attributes.connection_warming; + free_connections_pct = mysrvc->myhgc->attributes.free_connections_pct; + } + if (connection_warming == true) { unsigned int total_connections = mysrvc->ConnectionsFree->conns_length()+mysrvc->ConnectionsUsed->conns_length(); - unsigned int expected_warm_connections = mysql_thread___free_connections_pct*mysrvc->max_connections/100; + unsigned int expected_warm_connections = free_connections_pct*mysrvc->max_connections/100; if (total_connections < expected_warm_connections) { needs_warming = true; } @@ -3550,7 +3557,12 @@ void MySQL_HostGroups_Manager::drop_all_idle_connections() { //PtrArray *pa=mysrvc->ConnectionsFree->conns; MySrvConnList *mscl=mysrvc->ConnectionsFree; - while (mscl->conns_length() > mysql_thread___free_connections_pct*mysrvc->max_connections/100) { + int free_connections_pct = mysql_thread___free_connections_pct; + if (mysrvc->myhgc->attributes.configured == true) { + // mysql_hostgroup_attributes takes priority + free_connections_pct = mysrvc->myhgc->attributes.free_connections_pct; + } + while (mscl->conns_length() > free_connections_pct*mysrvc->max_connections/100) { MySQL_Connection *mc=mscl->remove(0); delete mc; } From dcf9a4a3623682441cdc30e544be3442e881312b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 29 Jan 2023 22:23:55 +0000 Subject: [PATCH 13/13] mysql_hostgroup_attributes.throttle_connections_per_sec Implementation of mysql_hostgroup_attributes.throttle_connections_per_sec If mysql_hostgroup_attributes is configured, mysql_hostgroup_attributes.throttle_connections_per_sec has higher priority than global variable mysql-throttle_connections_per_sec_to_hostgroup --- lib/MySQL_HostGroups_Manager.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 703254f51f..c607d34afa 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3169,7 +3169,12 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff _myhgc->new_connections_now = 0; } _myhgc->new_connections_now++; - if (_myhgc->new_connections_now > (unsigned int) mysql_thread___throttle_connections_per_sec_to_hostgroup) { + unsigned int throttle_connections_per_sec_to_hostgroup = (unsigned int) mysql_thread___throttle_connections_per_sec_to_hostgroup; + if (_myhgc->attributes.configured == true) { + // mysql_hostgroup_attributes takes priority + throttle_connections_per_sec_to_hostgroup = _myhgc->attributes.throttle_connections_per_sec; + } + if (_myhgc->new_connections_now > (unsigned int) throttle_connections_per_sec_to_hostgroup) { __sync_fetch_and_add(&MyHGM->status.server_connections_delayed, 1); return NULL; } else {