Refactoring cluster module mysql_servers

include/MySQL_HostGroups_Manager.h

@@ -66,6 +66,55 @@ using json = nlohmann::json;
 
 #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
 
+/*
+ * @brief Generates the 'runtime_mysql_servers' resultset exposed to other ProxySQL cluster members.
+ * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states
+ *  are by definition local transitory states, this is why a 'mysql_servers' table reconfiguration isn't
+ *  normally performed when servers are internally imposed with these statuses. This means, that propagating
+ *  this state to other cluster members is undesired behavior, and so it's generating a different checksum,
+ *  due to a server having this particular state, that will result in extra unnecessary fetching operations.
+ *  The query also filters out 'OFFLINE_HARD' servers, 'OFFLINE_HARD' is a local status which is equivalent to
+ *  a server no longer being part of the table (DELETED state). And so, they shouldn't be propagated.
+ *
+ *  For placing the query into a single line for debugging purposes:
+ *  ```
+ *  sed 's/^\t\+"//g; s/"\s\\$//g; s/\\"/"/g' /tmp/select.sql | paste -sd ''
+ *  ```
+ */
+#define MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS \
+	"SELECT " \
+		"hostgroup_id, hostname, port, gtid_port," \
+		"CASE status" \
+		" WHEN 0 THEN \"ONLINE\"" \
+		" WHEN 1 THEN \"ONLINE\"" \
+		" WHEN 2 THEN \"OFFLINE_SOFT\"" \
+		" WHEN 3 THEN \"OFFLINE_HARD\"" \
+		" WHEN 4 THEN \"ONLINE\" " \
+		"END status," \
+		"weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \
+	"FROM mysql_servers " \
+	"WHERE status != 3 " \
+	"ORDER BY hostgroup_id, hostname, port" \
+
+/**
+ * @brief Generates the 'mysql_servers_v2' resultset exposed to other ProxySQL cluster members.
+ * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config
+ *  ('mysql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin
+ *  interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also
+ *  filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration
+ *  statuses, they are local, transient status that ProxySQL uses during operation.
+ */
+#define MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS \
+	"SELECT " \
+		"hostgroup_id, hostname, port, gtid_port, " \
+		"CASE" \
+		" WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \
+		" ELSE status " \
+		"END AS status, " \
+		"weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \
+	"FROM main.mysql_servers " \
+	"WHERE status != \"OFFLINE_HARD\" " \
+	"ORDER BY hostgroup_id, hostname, port" \
 
 typedef std::unordered_map<std::uint64_t, void *> umap_mysql_errors;
 
@@ -446,12 +495,13 @@ class MySQL_HostGroups_Manager {
 #endif
 
 	enum HGM_TABLES {
-		MYSQL_SERVERS = 0,
+		MYSQL_SERVERS_V2 = 0,
 		MYSQL_REPLICATION_HOSTGROUPS,
 		MYSQL_GROUP_REPLICATION_HOSTGROUPS,
 		MYSQL_GALERA_HOSTGROUPS,
 		MYSQL_AWS_AURORA_HOSTGROUPS,
 		MYSQL_HOSTGROUP_ATTRIBUTES,
+		MYSQL_SERVERS,
 
 		__HGM_TABLES_SIZE
 	};
@@ -554,7 +604,7 @@ class MySQL_HostGroups_Manager {
 	/**
 	 * @brief This resultset holds the current values for 'runtime_mysql_servers' computed by either latest
 	 *  'commit' or fetched from another Cluster node. It's also used by ProxySQL_Admin to respond to the
-	 *  intercepted query 'CLUSTER_QUERY_MYSQL_SERVERS'.
+	 *  intercepted query 'CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS'.
 	 * @details This resultset can't right now just contain the value for 'incoming_mysql_servers' as with the
 	 *  rest of the intercepted resultset. This is due to 'runtime_mysql_servers' reconfigurations that can be
 	 *  triggered by monitoring actions like 'Galera' currently performs. These actions not only trigger status
@@ -605,6 +655,8 @@ class MySQL_HostGroups_Manager {
 	void generate_mysql_hostgroup_attributes_table();
 	SQLite3_result *incoming_hostgroup_attributes;
 
+	SQLite3_result* incoming_mysql_servers_v2;
+
 	std::thread *HGCU_thread;
 
 	std::thread *GTID_syncer_thread;
@@ -748,16 +800,26 @@ class MySQL_HostGroups_Manager {
 	void wrlock();
 	void wrunlock();
 	int servers_add(SQLite3_result *resultset);
-	bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0);
-	void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init);
-	void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables()
+	bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {},
+		SQLite3_result* mysql_servers_v2 = nullptr, const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {},
+		bool only_commit_runtime_mysql_servers = true, bool update_version = false);
+	void commit_update_checksums_from_tables();
+	void CUCFT1(const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables()
 
 	/**
 	 * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime.
 	 *  The store configuration is later used by Cluster to propagate current config.
 	 * @param The resulset to be stored replacing the current one.
 	 */
 	void save_runtime_mysql_servers(SQLite3_result *);
+
+	/**
+	 * @brief Store the resultset for the 'mysql_servers_v2' table.
+	 *  The store configuration is later used by Cluster to propagate current config.
+	 * @param The resulset to be stored replacing the current one.
+	 */
+	void save_mysql_servers_v2(SQLite3_result* s);
+
 	/**
 	 * @brief These setters/getter functions store and retrieve the currently hold resultset for the
 	 *  'incoming_*' table set that have been loaded to runtime. The store configuration is later used by
@@ -769,7 +831,20 @@ class MySQL_HostGroups_Manager {
 	SQLite3_result* get_current_mysql_table(const string& name);
 
 	SQLite3_result * execute_query(char *query, char **error);
-	SQLite3_result *dump_table_mysql(const string&);
+	/**
+	 * @brief Creates a resultset with the current full content of the target table.
+	 * @param string The target table. Valid values are:
+	 *   - "mysql_aws_aurora_hostgroups"
+	 *   - "mysql_galera_hostgroups"
+	 *   - "mysql_group_replication_hostgroups"
+	 *   - "mysql_replication_hostgroups"
+	 *   - "mysql_hostgroup_attributes"
+	 *   - "mysql_servers"
+	 *   - "cluster_mysql_servers"
+	 *   When targeting 'mysql_servers' table is purged and regenerated.
+	 * @return The generated resultset.
+	 */
+	SQLite3_result* dump_table_mysql(const string&);
 
 	/**
 	 * @brief Update the public member resulset 'mysql_servers_to_monitor'. This resulset should contain the latest
@@ -864,6 +939,11 @@ class MySQL_HostGroups_Manager {
 	void shutdown();
 	void unshun_server_all_hostgroups(const char * address, uint16_t port, time_t t, int max_wait_sec, unsigned int *skip_hid);
 	MySrvC* find_server_in_hg(unsigned int _hid, const std::string& addr, int port);
+
+private:
+	void update_hostgroup_manager_mappings();
+	uint64_t get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers = nullptr);
+	uint64_t get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2 = nullptr);
 };
 
 #endif /* __CLASS_MYSQL_HOSTGROUPS_MANAGER_H */

include/ProxySQL_Cluster.hpp

@@ -19,17 +19,20 @@
  * the queries issued for generating the checksum for each of the target modules, for simpler reasoning, they should
  * also represent the actual resultset being received when issuing them, since this resultset is used for computing the
  * 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules:
- *   - 'runtime_mysql_servers': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups',
- *     'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'.
+ *   - 'runtime_mysql_servers': tables 'mysql_servers'
  *   - 'runtime_mysql_users'.
  *   - 'runtime_mysql_query_rules'.
- *
+ *   - 'mysql_servers_v2': tables admin 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups',
+ *     'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'.
  * IMPORTANT: For further clarify this means that it's important that the actual resultset produced by the intercepted
  * query preserve the filtering and ordering expressed in this queries.
  */
 
 /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_servers'. See top comment for details. */
-#define CLUSTER_QUERY_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port"
+#define CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port"
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'mysql_servers_v2'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_SERVERS_V2 "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_v2 WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port"
 
 /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_replication_hostgroups'. See top comment for details. */
 #define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup"
@@ -189,6 +192,7 @@ class ProxySQL_Node_Entry {
 		ProxySQL_Checksum_Value_2 mysql_servers;
 		ProxySQL_Checksum_Value_2 mysql_users;
 		ProxySQL_Checksum_Value_2 proxysql_servers;
+		ProxySQL_Checksum_Value_2 mysql_servers_v2;
 	} checksums_values;
 	uint64_t global_checksum;
 };
@@ -279,7 +283,9 @@ class ProxySQL_Cluster_Nodes {
 	SQLite3_result * stats_proxysql_servers_checksums();
 	SQLite3_result * stats_proxysql_servers_metrics();
 	void get_peer_to_sync_mysql_query_rules(char **host, uint16_t *port, char** ip_address);
-	void get_peer_to_sync_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address);
+	void get_peer_to_sync_runtime_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address);
+	void get_peer_to_sync_mysql_servers_v2(char** host, uint16_t* port, char** peer_mysql_servers_v2_checksum, 
+		char** peer_runtime_mysql_servers_checksum, char** ip_address);
 	void get_peer_to_sync_mysql_users(char **host, uint16_t *port, char** ip_address);
 	void get_peer_to_sync_mysql_variables(char **host, uint16_t *port, char** ip_address);
 	void get_peer_to_sync_admin_variables(char **host, uint16_t* port, char** ip_address);
@@ -366,6 +372,31 @@ struct variable_type {
 	};
 };
 
+/**
+ * @brief Specifies the sync algorithm to use when pulling config from another ProxySQL cluster peer.
+ */
+enum class mysql_servers_sync_algorithm {
+	/**
+	 * @brief Sync 'runtime_mysql_servers' and 'mysql_server_v2' from remote peer.
+	 * @details This means that both config and runtime servers info are to be synced, in other words, both
+	 *  user promoted config and runtime changes performed by ProxySQL ('Monitor') in the remote peer will
+	 *  trigger the start of syncing operations.
+	 */
+	runtime_mysql_servers_and_mysql_servers_v2 = 1,
+	/**
+	 * @brief Sync only mysql_server_v2 (config) from the remote peer.
+	 * @details Since 'runtime_mysql_servers' isn't considered for fetching, only config changes promoted by
+	 *  the user in the remote peer will by acknowledge and trigger the start of a syncing operation.
+	 */
+	mysql_servers_v2 = 2,
+	/**
+	 * @brief Dependent on whether ProxySQL has been started with the -M flag.
+	 * @details If '-M' is not present, 'runtime_mysql_servers_and_mysql_servers_v2' is selected, otherwise
+	 *  'mysql_servers_v2' is chosen.
+	 */
+	auto_select = 3
+};
+
 /**
  * @brief Simple struct for holding a query, and three messages to report
  *  the progress of the query execution.
@@ -378,21 +409,23 @@ struct fetch_query {
 };
 
 class ProxySQL_Cluster {
-	private:
+private:
+	SQLite3DB* mydb;
 	pthread_mutex_t mutex;
 	std::vector<pthread_t> term_threads;
 	ProxySQL_Cluster_Nodes nodes;
-	char *cluster_username;
-	char *cluster_password;
+	char* cluster_username;
+	char* cluster_password;
 	struct {
-		std::array<prometheus::Counter*, p_cluster_counter::__size> p_counter_array {};
-		std::array<prometheus::Gauge*, p_cluster_gauge::__size> p_gauge_array {};
+		std::array<prometheus::Counter*, p_cluster_counter::__size> p_counter_array{};
+		std::array<prometheus::Gauge*, p_cluster_gauge::__size> p_gauge_array{};
 	} metrics;
 	int fetch_and_store(MYSQL* conn, const fetch_query& f_query, MYSQL_RES** result);
 	friend class ProxySQL_Node_Entry;
-	public:
+public:
 	pthread_mutex_t update_mysql_query_rules_mutex;
-	pthread_mutex_t update_mysql_servers_mutex;
+	pthread_mutex_t update_runtime_mysql_servers_mutex;
+	pthread_mutex_t update_mysql_servers_v2_mutex;
 	pthread_mutex_t update_mysql_users_mutex;
 	pthread_mutex_t update_mysql_variables_mutex;
 	pthread_mutex_t update_proxysql_servers_mutex;
@@ -407,7 +440,7 @@ class ProxySQL_Cluster {
 	 */
 	SQLite3_result* proxysql_servers_to_monitor;
 
-	char *admin_mysql_ifaces;
+	char* admin_mysql_ifaces;
 	int cluster_check_interval_ms;
 	int cluster_check_status_frequency;
 	int cluster_mysql_query_rules_diffs_before_sync;
@@ -417,6 +450,7 @@ class ProxySQL_Cluster {
 	int cluster_mysql_variables_diffs_before_sync;
 	int cluster_ldap_variables_diffs_before_sync;
 	int cluster_admin_variables_diffs_before_sync;
+	int cluster_mysql_servers_sync_algorithm;
 	bool cluster_mysql_query_rules_save_to_disk;
 	bool cluster_mysql_servers_save_to_disk;
 	bool cluster_mysql_users_save_to_disk;
@@ -428,7 +462,7 @@ class ProxySQL_Cluster {
 	~ProxySQL_Cluster();
 	void init() {};
 	void print_version();
-	void load_servers_list(SQLite3_result *r, bool _lock = true) {
+	void load_servers_list(SQLite3_result* r, bool _lock = true) {
 		nodes.load_servers_list(r, _lock);
 	}
 	void update_table_proxysql_servers_for_monitor(SQLite3_result* resultset) {
@@ -440,17 +474,17 @@ class ProxySQL_Cluster {
 
 		MySQL_Monitor::trigger_dns_cache_update();
 	}
-	void get_credentials(char **, char **);
-	void set_username(char *);
-	void set_password(char *);
-	void set_admin_mysql_ifaces(char *);
-	bool Update_Node_Metrics(char * _h, uint16_t _p, MYSQL_RES *_r, unsigned long long _response_time) {
+	void get_credentials(char**, char**);
+	void set_username(char*);
+	void set_password(char*);
+	void set_admin_mysql_ifaces(char*);
+	bool Update_Node_Metrics(char* _h, uint16_t _p, MYSQL_RES* _r, unsigned long long _response_time) {
 		return nodes.Update_Node_Metrics(_h, _p, _r, _response_time);
 	}
-	bool Update_Global_Checksum(char * _h, uint16_t _p, MYSQL_RES *_r) {
+	bool Update_Global_Checksum(char* _h, uint16_t _p, MYSQL_RES* _r) {
 		return nodes.Update_Global_Checksum(_h, _p, _r);
 	}
-	bool Update_Node_Checksums(char * _h, uint16_t _p, MYSQL_RES *_r = NULL) {
+	bool Update_Node_Checksums(char* _h, uint16_t _p, MYSQL_RES* _r = NULL) {
 		return nodes.Update_Node_Checksums(_h, _p, _r);
 	}
 	void Reset_Global_Checksums(bool lock) {
@@ -459,17 +493,19 @@ class ProxySQL_Cluster {
 	SQLite3_result *dump_table_proxysql_servers() {
 		return nodes.dump_table_proxysql_servers();
 	}
-	SQLite3_result * get_stats_proxysql_servers_checksums() {
+	SQLite3_result* get_stats_proxysql_servers_checksums() {
 		return nodes.stats_proxysql_servers_checksums();
 	}
-	SQLite3_result * get_stats_proxysql_servers_metrics() {
+	SQLite3_result* get_stats_proxysql_servers_metrics() {
 		return nodes.stats_proxysql_servers_metrics();
 	}
 	void p_update_metrics();
 	void thread_ending(pthread_t);
 	void join_term_thread();
 	void pull_mysql_query_rules_from_peer(const std::string& expected_checksum, const time_t epoch);
-	void pull_mysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch);
+	void pull_runtime_mysql_servers_from_peer(const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server);
+	void pull_mysql_servers_v2_from_peer(const mysql_servers_v2_checksum_t& peer_mysql_server_v2,
+		const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {}, bool fetch_runtime_mysql_servers = false);
 	void pull_mysql_users_from_peer(const std::string& expected_checksum, const time_t epoch);
 	/**
 	 * @brief Pulls from peer the specified global variables by the type parameter.

include/proxysql_admin.h

@@ -118,6 +118,7 @@ struct admin_metrics_map_idx {
 extern int admin__web_verbosity;
 
 struct incoming_servers_t {
+	SQLite3_result* incoming_mysql_servers_v2 = NULL;
 	SQLite3_result* runtime_mysql_servers = NULL;
 	SQLite3_result* incoming_replication_hostgroups = NULL;
 	SQLite3_result* incoming_group_replication_hostgroups = NULL;
@@ -126,9 +127,27 @@ struct incoming_servers_t {
 	SQLite3_result* incoming_hostgroup_attributes = NULL;
 
 	incoming_servers_t();
-	incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*);
+	incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*);
 };
 
+// Separate structs for runtime mysql server and mysql server v2 to avoid human error
+struct runtime_mysql_servers_checksum_t {
+	std::string checksum;
+	time_t epoch;
+
+	runtime_mysql_servers_checksum_t();
+	runtime_mysql_servers_checksum_t(const std::string& checksum, time_t epoch);
+};
+
+struct mysql_servers_v2_checksum_t {
+	std::string checksum;
+	time_t epoch;
+
+	mysql_servers_v2_checksum_t();
+	mysql_servers_v2_checksum_t(const std::string& checksum, time_t epoch);
+};
+//
+
 class ProxySQL_Admin {
 	private:
 	volatile int main_shutdown;
@@ -186,6 +205,7 @@ class ProxySQL_Admin {
 		int cluster_mysql_variables_diffs_before_sync;
 		int cluster_admin_variables_diffs_before_sync;
 		int cluster_ldap_variables_diffs_before_sync;
+		int cluster_mysql_servers_sync_algorithm;
 		bool cluster_mysql_query_rules_save_to_disk;
 		bool cluster_mysql_servers_save_to_disk;
 		bool cluster_mysql_users_save_to_disk;
@@ -410,7 +430,8 @@ class ProxySQL_Admin {
 //	void flush_admin_variables__from_disk_to_memory(); // commented in 2.3 because unused
 	void flush_admin_variables__from_memory_to_disk();
 	void flush_ldap_variables__from_memory_to_disk();
-	void load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers = {}, const std::string& checksum = "", const time_t epoch = 0);
+	void load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers = {}, const runtime_mysql_servers_checksum_t& peer_runtime_mysql_server = {},
+		const mysql_servers_v2_checksum_t& peer_mysql_server_v2 = {});
 	void save_mysql_servers_from_runtime();
 	/**
 	 * @brief Performs the load to runtime of the current configuration in 'main' for 'mysql_query_rules' and